DOCSTOOLS-884: add source path to md metadata when merging

package.json

@@ -61,6 +61,7 @@
     "@types/react": "^16.9.23",
     "@types/react-dom": "^16.9.5",
     "@types/shelljs": "^0.8.7",
+    "@types/yaml-front-matter": "^4.1.0",
     "@types/yargs": "^15.0.4",
     "@typescript-eslint/eslint-plugin": "^2.27.0",
     "@typescript-eslint/parser": "^2.29.0",
@@ -85,7 +86,8 @@
     "ts-jest": "^27.0.5",
     "typescript": "^3.9.2",
     "webpack": "^4.42.1",
-    "webpack-cli": "^3.3.11"
+    "webpack-cli": "^3.3.11",
+    "yaml-front-matter": "^4.1.1"
   },
   "husky": {
     "hooks": {

src/models.ts

@@ -111,16 +111,18 @@ export interface Contributors {
 }
 
 export interface FileData {
-    tmpInputFilePath: string;
-    inputFolderPathLength: number;
-    fileContent: string;
+    tmpInputFilePath?: string;
+    inputFolderPathLength?: number;
+    fileContent?: string;
+    sourcePath?: string;
 }
 
 export interface MetaDataOptions {
     fileData: FileData;
     isContributorsEnabled?: boolean;
     vcsConnector?: VCSConnector;
     addSystemMeta?: boolean;
+    addSourcePath?: boolean;
 }
 
 export interface PluginOptions {

src/services/metadata.ts

@@ -1,3 +1,6 @@
+import {safeLoadFront} from 'yaml-front-matter';
+import {dump} from 'js-yaml';
+
 import {VCSConnector} from '../vcs-connector/connector-models';
 import {Metadata, MetaDataOptions} from '../models';
 import {getAuthorDetails, updateAuthorMetadataString} from './authors';
@@ -9,48 +12,74 @@ async function getContentWithUpdatedMetadata(
     options?: MetaDataOptions,
     systemVars?: unknown,
 ): Promise<string> {
-    if (!options?.isContributorsEnabled && (!options?.addSystemMeta || !systemVars)) {
+    let result;
+
+    result = getContentWithUpdatedStaticMetadata(fileContent, options, systemVars);
+    result = await getContentWithUpdatedDynamicMetadata(result, options);
+
+    return result;
+}
+
+function getContentWithUpdatedStaticMetadata(
+    fileContent: string,
+    options?: MetaDataOptions,
+    systemVars?: unknown,
+): string {
+    if (!options || (!options?.addSystemMeta || !systemVars) && !options?.addSourcePath) {
         return fileContent;
     }
-    // Search by format:
-    // ---
-    // metaName1: metaValue1
-    // metaName2: meta value2
-    // incorrectMetadata
-    // ---
-    const regexpMetadata = '(?<=-{3}\\r?\\n)((.*\\r?\\n)*)(?=-{3}\\r?\\n)';
-    // Search by format:
-    // ---
-    // main content 123
-    const regexpFileContent = '---((.*[\r?\n]*)*)';
-
-    const regexpParseFileContent = new RegExp(`${regexpMetadata}${regexpFileContent}`, 'gm');
-    const matches = regexpParseFileContent.exec(fileContent);
-
-    const newMetadatas = [];
-
-    if (options) {
-        const {isContributorsEnabled, addSystemMeta} = options;
-
-        if (addSystemMeta && systemVars && isObject(systemVars)) {
-            newMetadatas.push(getSystemVarsMetadataString(systemVars));
-        }
-
-        if (isContributorsEnabled) {
-            newMetadatas.push(await getContributorsMetadataString(options, fileContent));
-        }
-
-        if (matches && matches.length > 0) {
-            const [, fileMetadata, , fileMainContent] = matches;
-            let updatedDefaultMetadata = '';
-
-            updatedDefaultMetadata = await updateAuthorMetadataString(fileMetadata, options.vcsConnector);
-
-            return `${getUpdatedMetadataString(newMetadatas, updatedDefaultMetadata)}${fileMainContent}`;
-        }
+
+    const matches = matchMetadata(fileContent);
+    const newMetadatas: string[] = [];
+
+    const {addSystemMeta, addSourcePath, fileData} = options;
+
+    if (addSystemMeta && systemVars && isObject(systemVars)) {
+        newMetadatas.push(getSystemVarsMetadataString(systemVars));
+    }
+
+    if (addSourcePath && fileData.sourcePath) {
+        const sourcePathMetadataString = `sourcePath: ${fileData.sourcePath}`;
+        newMetadatas.push(sourcePathMetadataString);
     }
 
-    return `${getUpdatedMetadataString(newMetadatas)}${fileContent}`;
+    const {fileMetadata, fileMainContent} = matches;
+
+    return `${getUpdatedMetadataString(newMetadatas, fileMetadata)}${fileMainContent}`;
+}
+
+async function getContentWithUpdatedDynamicMetadata(
+    fileContent: string,
+    options?: MetaDataOptions,
+): Promise<string> {
+    if (!options || !options?.isContributorsEnabled) {
+        return fileContent;
+    }
+
+    const matches = matchMetadata(fileContent);
+    const newMetadatas: string[] = [];
+
+    const {isContributorsEnabled} = options;
+
+    if (isContributorsEnabled) {
+        newMetadatas.push(await getContributorsMetadataString(options, fileContent));
+    }
+
+    const {fileMetadata, fileMainContent} = matches;
+
+    const updatedDefaultMetadata = await updateAuthorMetadataString(fileMetadata, options.vcsConnector);
+
+    return `${getUpdatedMetadataString(newMetadatas, updatedDefaultMetadata)}${fileMainContent}`;
+}
+
+function matchMetadata(fileContent: string) {
+    const {__content: fileMainContent, ...metadata} = safeLoadFront(fileContent);
+    const fileMetadata = Object.keys(metadata).length ? dump(metadata) : '';
+
+    return {
+        fileMainContent,
+        fileMetadata,
+    };
 }
 
 async function getContributorsMetadataString(options: MetaDataOptions, fileContent: string): Promise<string> {
@@ -74,7 +103,9 @@ function getUpdatedMetadataString(newMetadatas: string[], defaultMetadata = ''):
     const metadataBorder = `---${metadataСarriage}`;
 
     const newMetadata = newMetadatas.join(metadataСarriage);
-    const updatedMetadata = `${defaultMetadata}${metadataСarriage}${newMetadata}${metadataСarriage}`;
+    const preparedDefaultMetadata = defaultMetadata.trimRight();
+    const defaultMetadataСarriage = preparedDefaultMetadata ? metadataСarriage : '';
+    const updatedMetadata = `${preparedDefaultMetadata}${defaultMetadataСarriage}${newMetadata}${metadataСarriage}`;
 
     return `${metadataBorder}${updatedMetadata}${metadataBorder}`;
 }
@@ -130,5 +161,6 @@ function getSystemVarsMetadataString(systemVars: object) {
 
 export {
     getContentWithUpdatedMetadata,
+    getContentWithUpdatedStaticMetadata,
     getUpdatedMetadata,
 };

src/services/tocs.ts

@@ -9,6 +9,7 @@ import {getSinglePageAnchorId} from '@doc-tools/transform/lib/utilsFS';
 import {bold} from 'chalk';
 
 import {ArgvService, PresetService} from './index';
+import {getContentWithUpdatedStaticMetadata} from './metadata';
 import {YfmToc} from '../models';
 import {Stage, SINGLE_PAGE_FOLDER, IncludeMode} from '../constants';
 import {isExternalHref} from '../utils';
@@ -200,6 +201,8 @@ function _normalizeHref(href: string): string {
  * @private
  */
 function _copyTocDir(tocPath: string, destDir: string) {
+    const {input: inputFolderPath} = ArgvService.getConfig();
+
     const {dir: tocDir} = parse(tocPath);
     const files: string[] = walkSync(tocDir, {
         globs: ['**/*.*'],
@@ -210,9 +213,24 @@ function _copyTocDir(tocPath: string, destDir: string) {
     files.forEach((relPath) => {
         const from = resolve(tocDir, relPath);
         const to = resolve(destDir, relPath);
+        const fileExtension = extname(relPath);
+        const isMdFile = fileExtension === '.md';
 
         shell.mkdir('-p', parse(to).dir);
-        copyFileSync(from, to);
+
+        if (isMdFile) {
+            const fileContent = readFileSync(from, 'utf8');
+            const sourcePath = relative(inputFolderPath, from);
+            const fileData = {sourcePath};
+            const updatedFileContent = getContentWithUpdatedStaticMetadata(fileContent, {
+                fileData,
+                addSourcePath: true,
+            });
+
+            writeFileSync(to, updatedFileContent);
+        } else {
+            copyFileSync(from, to);
+        }
     });
 }
 
@@ -320,7 +338,9 @@ function _replaceIncludes(items: YfmToc[], tocDir: string, sourcesDir: string, v
                     includedInlineItems = includedTocItems;
                 }
             } catch (err) {
-                log.error(`Error while including toc: ${bold(includeTocPath)} to ${bold(join(tocDir, 'toc.yaml'))}`);
+                const message = `Error while including toc: ${bold(includeTocPath)} to ${bold(join(tocDir, 'toc.yaml'))}`;
+                console.log(message, err);
+                log.error(message);
                 return acc;
             } finally {
                 delete item.include;

src/steps/processPages.ts

@@ -175,8 +175,9 @@ async function preparingPagesByOutputFormat(
                 return;
         }
     } catch (e) {
-        console.log(e);
-        log.error(` No such file or has no access to ${bold(resolvedPathToFile)}`);
+        const message = `No such file or has no access to ${bold(resolvedPathToFile)}`;
+        console.log(message, e);
+        log.error(message);
     }
 }
 

tests/include-toc.test.ts

@@ -34,4 +34,20 @@ describe('Include toc', () => {
             expect(expectedContent).toEqual(outputContent);
         }
     });
+
+    test('Nested toc inclusions with mixed including modes', () => {
+        const testRootPath = './tests/mocks/include-toc/test3';
+        const {inputPath, outputPath, expectedOutputPath} = getTestPaths(testRootPath);
+        runYfmDocs(inputPath, outputPath);
+
+        const compareResult = compareDirectories(outputPath, expectedOutputPath);
+
+        if (typeof compareResult === 'boolean') {
+            expect(true).toEqual(compareResult);
+        } else {
+            const {expectedContent, outputContent} = compareResult;
+
+            expect(expectedContent).toEqual(outputContent);
+        }
+    });
 });

tests/mocks/include-toc/test2/expected-output/fileA.md

@@ -1 +1,6 @@
+---
+title: File A
+sourcePath: folder/fileA.md
+---
+
 # File A

tests/mocks/include-toc/test2/expected-output/fileB.md

@@ -1 +1,12 @@
+---
+sourcePath: folder/fileB.md
+---
 # File B
+
+Lorem Ipsum is simply dummy text of the printing and typesetting industry.
+
+---
+
+Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book.
+
+---

tests/mocks/include-toc/test2/expected-output/fileC.md

@@ -1 +1,4 @@
+---
+sourcePath: folder/folder/fileC.md
+---
 # File C

tests/mocks/include-toc/test2/expected-output/folder/fileA.md

@@ -1 +1,4 @@
+---
+title: File A
+---
 # File A

tests/mocks/include-toc/test2/expected-output/folder/fileB.md

@@ -1 +1,9 @@
 # File B
+
+Lorem Ipsum is simply dummy text of the printing and typesetting industry.
+
+---
+
+Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book.
+
+---

tests/mocks/include-toc/test2/expected-output/folder/fileC.md

@@ -1 +1,4 @@
+---
+sourcePath: folder/folder/fileC.md
+---
 # File C

tests/mocks/include-toc/test2/input/folder/fileA.md

@@ -1 +1,4 @@
+---
+title: File A
+---
 # File A

tests/mocks/include-toc/test2/input/folder/fileB.md

@@ -1 +1,9 @@
 # File B
+
+Lorem Ipsum is simply dummy text of the printing and typesetting industry.
+
+---
+
+Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book.
+
+---

tests/mocks/include-toc/test3/expected-output/product1/_includes/inc.md

@@ -0,0 +1,4 @@
+---
+sourcePath: product1/core/_includes/inc.md
+---
+This is the core include.

tests/mocks/include-toc/test3/expected-output/product1/article1.md

@@ -0,0 +1,6 @@
+---
+sourcePath: product1/core/article1.md
+---
+This is the core content of Article 1.
+
+{% include [x](_includes/inc.md) %}

tests/mocks/include-toc/test3/expected-output/product1/toc.yaml

@@ -0,0 +1,5 @@
+title: Product 1 title
+items:
+  - name: Article1
+    href: article1.md
+base: product1