asset mapper file creation

packages/contentstack-import-setup/src/config/index.ts

@@ -31,11 +31,12 @@ const config: DefaultConfig = {
     assets: {
       dirName: 'assets',
       fileName: 'assets.json',
+      fetchConcurrency: 5,
     },
     'content-types': {
       dirName: 'content_types',
       fileName: 'content_types.json',
-      dependencies: ['extensions', 'taxonomies'],
+      dependencies: ['assets', 'extensions', 'taxonomies'],
     },
     entries: {
       dirName: 'entries',
@@ -71,6 +72,7 @@ const config: DefaultConfig = {
       ],
     },
   },
+  fetchConcurrency: 5,
 };
 
 export default config;

packages/contentstack-import-setup/src/import/modules/assets.ts

@@ -0,0 +1,121 @@
+import * as chalk from 'chalk';
+import { log, fsUtil } from '../../utils';
+import { join } from 'path';
+import { AssetRecord, ImportConfig, ModuleClassParams } from '../../types';
+import { isEmpty, orderBy, values } from 'lodash';
+import { formatError, FsUtility } from '@contentstack/cli-utilities';
+import BaseImportSetup from './base-setup';
+
+export default class AssetImportSetup extends BaseImportSetup {
+  private assetsFilePath: string;
+  private assetUidMapper: Record<string, string>;
+  private assetUrlMapper: Record<string, string>;
+  private duplicateAssets: Record<string, string>;
+  private assetsConfig: ImportConfig['modules']['assets'];
+  private mapperDirPath: string;
+  private assetsFolderPath: string;
+  private assetUidMapperPath: string;
+  private assetUrlMapperPath: string;
+  private duplicateAssetPath: string;
+
+  constructor({ config, stackAPIClient, dependencies }: ModuleClassParams) {
+    super({ config, stackAPIClient, dependencies });
+    this.assetsFolderPath = join(this.config.contentDir, 'assets');
+    this.assetsFilePath = join(this.config.contentDir, 'assets', 'assets.json');
+    this.assetsConfig = config.modules.assets;
+    this.mapperDirPath = join(this.config.backupDir, 'mapper', 'assets');
+    this.assetUidMapperPath = join(this.config.backupDir, 'mapper', 'assets', 'uid-mapping.json');
+    this.assetUrlMapperPath = join(this.config.backupDir, 'mapper', 'assets', 'url-mapping.json');
+    this.duplicateAssetPath = join(this.config.backupDir, 'mapper', 'assets', 'duplicate-assets.json');
+    this.assetUidMapper = {};
+    this.assetUrlMapper = {};
+    this.duplicateAssets = {};
+  }
+
+  /**
+   * Start the asset import setup
+   * This method reads the assets from the content folder and generates a mapper file
+   * @returns {Promise<void>}
+   */
+  async start() {
+    try {
+      fsUtil.makeDirectory(this.mapperDirPath);
+      await this.fetchAndMapAssets();
+      log(this.config, `Generated required setup files for asset`, 'success');
+    } catch (error) {
+      log(this.config, `Error generating asset mapper: ${formatError(error)}`, 'error');
+    }
+  }
+
+  /**
+   * @method importAssets
+   * @param {boolean} isVersion boolean
+   * @returns {Promise<void>} Promise<void>
+   */
+  async fetchAndMapAssets(): Promise<void> {
+    const processName = 'mapping assets';
+    const indexFileName = 'assets.json';
+    const basePath = this.assetsFolderPath;
+    const fs = new FsUtility({ basePath, indexFileName });
+    const indexer = fs.indexFileContent;
+    const indexerCount = values(indexer).length;
+
+    const onSuccess = ({
+      response: { items = [] as AssetRecord[] } = {},
+      apiData: { uid, url, title } = undefined,
+    }: any) => {
+      if (items.length === 1) {
+        this.assetUidMapper[uid] = items[0].uid;
+        this.assetUrlMapper[url] = items[0].url;
+        log(this.config, `Mapped asset: '${title}'`, 'info');
+      } else if (items.length > 1) {
+        this.duplicateAssets[uid] = items.map((asset: any) => {
+          return { uid: asset.uid, title: asset.title, url: asset.url };
+        });
+        log(this.config, `Multiple assets found with title '${title}'`, 'error');
+      } else {
+        log(this.config, `Asset with title '${title}' not found in the stack!`, 'error');
+      }
+    };
+    const onReject = ({ error, apiData: { title } = undefined }: any) => {
+      log(this.config, `${title} asset mapping failed.!`, 'error');
+      log(this.config, formatError(error), 'error');
+    };
+
+    /* eslint-disable @typescript-eslint/no-unused-vars, guard-for-in */
+    for (const index in indexer) {
+      const chunk = await fs.readChunkFiles.next().catch((error) => {
+        log(this.config, error, 'error');
+      });
+
+      if (chunk) {
+        let apiContent = orderBy(values(chunk as Record<string, any>[]), '_version');
+
+        await this.makeConcurrentCall(
+          {
+            apiContent,
+            processName,
+            indexerCount,
+            currentIndexer: +index,
+            apiParams: {
+              reject: onReject,
+              resolve: onSuccess,
+              entity: 'fetch-assets',
+              includeParamOnCompletion: true,
+            },
+            concurrencyLimit: this.assetsConfig.fetchConcurrency,
+          },
+          undefined,
+        );
+      }
+    }
+
+    if (!isEmpty(this.assetUidMapper) || !isEmpty(this.assetUrlMapper)) {
+      fsUtil.writeFile(this.assetUidMapperPath, this.assetUidMapper);
+      fsUtil.writeFile(this.assetUrlMapperPath, this.assetUrlMapper);
+    }
+    if (!isEmpty(this.duplicateAssets)) {
+      fsUtil.writeFile(this.duplicateAssetPath, this.duplicateAssets);
+    }
+  }
+}

packages/contentstack-import-setup/src/import/modules/base-setup.ts

@@ -1,5 +1,6 @@
 import { log, fsUtil } from '../../utils';
-import { ImportConfig, ModuleClassParams } from '../../types';
+import { ApiOptions, CustomPromiseHandler, EnvType, ImportConfig, ModuleClassParams } from '../../types';
+import { chunk, entries, isEmpty, isEqual, last } from 'lodash';
 
 export default class BaseImportSetup {
   public config: ImportConfig;
@@ -30,4 +31,178 @@ export default class BaseImportSetup {
       }
     }
   }
+
+  /**
+   * @method delay
+   * @param {number} ms number
+   * @returns {Promise} Promise<void>
+   */
+  delay(ms: number): Promise<void> {
+    /* eslint-disable no-promise-executor-return */
+    return new Promise((resolve) => setTimeout(resolve, ms <= 0 ? 0 : ms));
+  }
+
+  /**
+   * @method makeConcurrentCall
+   * @param {Record<string, any>} env EnvType
+   * @param {CustomPromiseHandler} promisifyHandler CustomPromiseHandler
+   * @param {boolean} logBatchCompletionMsg boolean
+   * @returns {Promise} Promise<void>
+   */
+  makeConcurrentCall(
+    env: EnvType,
+    promisifyHandler?: CustomPromiseHandler,
+    logBatchCompletionMsg = true,
+  ): Promise<void> {
+    const {
+      apiParams,
+      apiContent,
+      processName,
+      indexerCount,
+      currentIndexer,
+      concurrencyLimit = this.config.fetchConcurrency,
+    } = env;
+
+    /* eslint-disable no-async-promise-executor */
+    return new Promise(async (resolve) => {
+      let batchNo = 0;
+      let isLastRequest = false;
+      const batches: Array<Record<string, any>> = chunk(apiContent, concurrencyLimit);
+
+      /* eslint-disable no-promise-executor-return */
+      if (isEmpty(batches)) return resolve();
+
+      for (const [batchIndex, batch] of entries(batches)) {
+        batchNo += 1;
+        const allPromise = [];
+        const start = Date.now();
+
+        for (const [index, element] of entries(batch)) {
+          let promise = Promise.resolve();
+          isLastRequest = isEqual(last(batch as ArrayLike<any>), element) && isEqual(last(batches), batch);
+
+          if (promisifyHandler instanceof Function) {
+            promise = promisifyHandler({
+              apiParams,
+              isLastRequest,
+              element,
+              index: Number(index),
+              batchIndex: Number(batchIndex),
+            });
+          } else if (apiParams) {
+            apiParams.apiData = element;
+            promise = this.makeAPICall(apiParams, isLastRequest);
+          }
+
+          allPromise.push(promise);
+        }
+
+        /* eslint-disable no-await-in-loop */
+        await Promise.allSettled(allPromise);
+
+        /* eslint-disable no-await-in-loop */
+        await this.logMsgAndWaitIfRequired(
+          processName,
+          start,
+          batches.length,
+          batchNo,
+          logBatchCompletionMsg,
+          indexerCount,
+          currentIndexer,
+        );
+
+        if (isLastRequest) resolve();
+      }
+    });
+  }
+
+  /**
+   * @method logMsgAndWaitIfRequired
+   * @param {string} processName string
+   * @param {number} start number
+   * @param {number} batchNo - number
+   * @returns {Promise} Promise<void>
+   */
+  async logMsgAndWaitIfRequired(
+    processName: string,
+    start: number,
+    totelBatches: number,
+    batchNo: number,
+    logBatchCompletionMsg = true,
+    indexerCount?: number,
+    currentIndexer?: number,
+  ): Promise<void> {
+    const end = Date.now();
+    const exeTime = end - start;
+
+    if (logBatchCompletionMsg) {
+      let batchMsg = '';
+      // info: Batch No. 20 of import assets is complete
+      if (currentIndexer) batchMsg += `Current chunk processing is (${currentIndexer}/${indexerCount})`;
+
+      log(this.config, `Batch No. (${batchNo}/${totelBatches}) of ${processName} is complete`, 'success');
+    }
+
+    // if (this.config.modules.assets.displayExecutionTime) {
+    //   console.log(
+    //     `Time taken to execute: ${exeTime} milliseconds; wait time: ${
+    //       exeTime < 1000 ? 1000 - exeTime : 0
+    //     } milliseconds`,
+    //   );
+    // }
+
+    if (exeTime < 1000) await this.delay(1000 - exeTime);
+  }
+
+  /**
+   * @method makeAPICall
+   * @param {Record<string, any>} apiOptions - Api related params
+   * @param {Record<string, any>} isLastRequest - Boolean
+   * @return {Promise} Promise<void>
+   */
+  makeAPICall(apiOptions: ApiOptions, isLastRequest = false): Promise<void> {
+    if (apiOptions.serializeData instanceof Function) {
+      apiOptions = apiOptions.serializeData(apiOptions);
+    }
+
+    const { uid, entity, reject, resolve, apiData, additionalInfo = {}, includeParamOnCompletion } = apiOptions;
+
+    const onSuccess = (response: any) =>
+      resolve({
+        response,
+        isLastRequest,
+        additionalInfo,
+        apiData: includeParamOnCompletion ? apiData : undefined,
+      });
+    const onReject = (error: Error) =>
+      reject({
+        error,
+        isLastRequest,
+        additionalInfo,
+        apiData: includeParamOnCompletion ? apiData : undefined,
+      });
+
+    if (!apiData) {
+      return Promise.resolve();
+    }
+    switch (entity) {
+      case 'fetch-assets':
+        return this.stackAPIClient
+          .asset()
+          .query({
+            query: {
+              $and: [
+                { file_size: Number(apiData.file_size) },
+                { filename: apiData.filename },
+                { title: apiData.title },
+              ],
+            },
+          })
+          .find()
+          .then(onSuccess)
+          .catch(onReject);
+      default:
+        return Promise.resolve();
+    }
+  }
 }

packages/contentstack-import-setup/src/import/modules/extensions.ts

@@ -3,6 +3,7 @@ import { log, fsUtil } from '../../utils';
 import { join } from 'path';
 import { ImportConfig, ModuleClassParams } from '../../types';
 import { isEmpty } from 'lodash';
+import { formatError } from '@contentstack/cli-utilities';
 
 export default class ExtensionImportSetup {
   private config: ImportConfig;
@@ -48,12 +49,12 @@ export default class ExtensionImportSetup {
 
         await fsUtil.writeFile(this.extUidMapperPath, this.extensionMapper);
 
-        log(this.config, `Generate required setup files for extension`, 'success');
+        log(this.config, `Generated required setup files for extension`, 'success');
       } else {
         log(this.config, 'No extensions found in the content folder!', 'error');
       }
     } catch (error) {
-      log(this.config, `Error generating extension mapper: ${error.message}`, 'error');
+      log(this.config, `Error generating extension mapper: ${formatError(error)}`, 'error');
     }
   }
 

packages/contentstack-import-setup/src/types/default-config.ts

@@ -22,6 +22,7 @@ export default interface DefaultConfig {
       dirName: string;
       fileName: string;
       dependencies?: Modules[];
+      fetchConcurrency: number;
     };
     'content-types': {
       dirName: string;
@@ -50,4 +51,5 @@ export default interface DefaultConfig {
       invalidKeys: string[];
     };
   };
+  fetchConcurrency: number;
 }