DX - 523 - Audit and Audit Fix for Mandatory Fields

packages/contentstack-audit/src/audit-base-command.ts

@@ -55,6 +55,7 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
       missingCtRefsInExtensions,
       missingCtRefsInWorkflow,
       missingSelectFeild,
+      missingMandatoryFields,
     } = await this.scanAndFix();
 
     this.showOutputOnScreen([
@@ -65,7 +66,9 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
     this.showOutputOnScreenWorkflowsAndExtension([{ module: 'Extensions', missingRefs: missingCtRefsInExtensions }]);
     this.showOutputOnScreenWorkflowsAndExtension([{ module: 'Workflows', missingRefs: missingCtRefsInWorkflow }]);
     this.showOutputOnScreenWorkflowsAndExtension([{ module: 'Entries Select Field', missingRefs: missingSelectFeild }]);
-
+    this.showOutputOnScreenWorkflowsAndExtension([
+      { module: 'Entries Mandatory Field', missingRefs: missingMandatoryFields },
+    ]);
     if (
       !isEmpty(missingCtRefs) ||
       !isEmpty(missingGfRefs) ||
@@ -117,7 +120,9 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
       missingCtRefsInExtensions,
       missingCtRefsInWorkflow,
       missingSelectFeild,
-      missingEntry;
+      missingEntry,
+      missingMandatoryFields;
+
     for (const module of this.sharedConfig.flags.modules || this.sharedConfig.modules) {
       print([
         {
@@ -148,9 +153,12 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
           missingEntry = await new Entries(cloneDeep(constructorParam)).run();
           missingEntryRefs = missingEntry.missingEntryRefs ?? {};
           missingSelectFeild = missingEntry.missingSelectFeild ?? {};
+          missingMandatoryFields = missingEntry.missingMandatoryFields ?? {};
           await this.prepareReport(module, missingEntryRefs);
 
-          await this.prepareReport(`entries_Select_feild`, missingSelectFeild);
+          await this.prepareReport(`Entries_Select_feild`, missingSelectFeild);
+
+          await this.prepareReport('Entries_Mandatory_feild', missingMandatoryFields);
           break;
         case 'workflows':
           missingCtRefsInWorkflow = await new Workflows({
@@ -189,6 +197,7 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
       missingCtRefsInExtensions,
       missingCtRefsInWorkflow,
       missingSelectFeild,
+      missingMandatoryFields,
     };
   }
 
@@ -335,17 +344,20 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
       missingRefs = Object.values(missingRefs).flat();
       const tableKeys = Object.keys(missingRefs[0]);
       const arrayOfObjects = tableKeys.map((key) => {
-        if (
-         config.OutputTableKeys.includes(key)
-        ) {
+        if (config.OutputTableKeys.includes(key)) {
           return {
             [key]: {
               minWidth: 7,
               header: key,
               get: (row: Record<string, unknown>) => {
                 if (key === 'fixStatus') {
                   return chalk.green(typeof row[key] === 'object' ? JSON.stringify(row[key]) : row[key]);
-                } else if (key === 'content_types' || key === 'branches' || key === 'missingCTSelectFieldValues') {
+                } else if (
+                  key === 'content_types' ||
+                  key === 'branches' ||
+                  key === 'missingCTSelectFieldValues' ||
+                  key === 'missingFieldUid'
+                ) {
                   return chalk.red(typeof row[key] === 'object' ? JSON.stringify(row[key]) : row[key]);
                 } else {
                   return chalk.white(typeof row[key] === 'object' ? JSON.stringify(row[key]) : row[key]);
@@ -374,7 +386,7 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
    * @returns The function `prepareReport` returns a Promise that resolves to `void`.
    */
   prepareReport(
-    moduleName: keyof typeof config.moduleConfig | 'entries_Select_feild',
+    moduleName: keyof typeof config.moduleConfig | keyof typeof config.ReportTitleForEntries,
     listOfMissingRefs: Record<string, any>,
   ): Promise<void> {
     if (isEmpty(listOfMissingRefs)) return Promise.resolve(void 0);
@@ -401,7 +413,7 @@ export abstract class AuditBaseCommand extends BaseCommand<typeof AuditBaseComma
    * @returns The function `prepareCSV` returns a Promise that resolves to `void`.
    */
   prepareCSV(
-    moduleName: keyof typeof config.moduleConfig | 'entries_Select_feild',
+    moduleName: keyof typeof config.moduleConfig | keyof typeof config.ReportTitleForEntries,
     listOfMissingRefs: Record<string, any>,
   ): Promise<void> {
     const csvPath = join(this.sharedConfig.reportPath, `${moduleName}.csv`);

packages/contentstack-audit/src/config/index.ts

@@ -55,7 +55,7 @@ const config = {
     ],
   },
   //These keys will be used output the modules with issues and fixes on console
-  OutputTableKeys : [
+  OutputTableKeys: [
     'title',
     'name',
     'uid',
@@ -69,7 +69,13 @@ const config = {
     'treeStr',
     'missingCTSelectFieldValues',
     'min_instance',
-  ]
+    'missingFieldUid',
+    'isPublished',
+  ],
+  ReportTitleForEntries: {
+    Entries_Select_feild: 'Entries_Select_feild',
+    Entries_Mandatory_feild: 'Entries_Mandatory_feild',
+  },
 };
 
 export default config;

packages/contentstack-audit/src/messages/index.ts

@@ -46,6 +46,8 @@ const auditFixMsg = {
   EMPTY_FIX_MSG: 'Successfully removed the empty field/block found at {path} from the schema.',
   AUDIT_FIX_CMD_DESCRIPTION: 'Perform audits and fix possible errors in the exported Contentstack data.',
   WF_FIX_MSG: 'Successfully removed the workflow {uid} named {name}.',
+  ENTRY_MANDATORY_FIELD_FIX: `Removing the publish details from entry uid '{uid}' from locale '{locale}'`,
+  ENTRY_SELECT_FIELD_FIX: `Adding the value '{value}' in select field of uid '{uid}'`
 };
 
 const messages: typeof errors &

packages/contentstack-audit/src/modules/entries.ts

@@ -8,7 +8,7 @@ import { existsSync, readFileSync, writeFileSync } from 'fs';
 
 import auditConfig from '../config';
 import ContentType from './content-types';
-import { $t, auditMsg, commonMsg } from '../messages';
+import { $t, auditFixMsg, auditMsg, commonMsg } from '../messages';
 import {
   LogFn,
   Locale,
@@ -56,6 +56,7 @@ export default class Entries {
   protected entries!: Record<string, EntryStruct>;
   protected missingRefs: Record<string, any> = {};
   protected missingSelectFeild: Record<string, any> = {};
+  protected missingMandatoryFields: Record<string, any> = {};
   public entryMetaData: Record<string, any>[] = [];
   public moduleName: keyof typeof auditConfig.moduleConfig = 'entries';
   public isEntryWithoutTitleField: boolean = false;
@@ -109,11 +110,35 @@ export default class Entries {
             if (!this.missingSelectFeild[this.currentUid]) {
               this.missingSelectFeild[this.currentUid] = [];
             }
+
+            if (!this.missingMandatoryFields[this.currentUid]) {
+              this.missingMandatoryFields[this.currentUid] = [];
+            }
             if (this.fix) {
               this.removeMissingKeysOnEntry(ctSchema.schema as ContentTypeSchemaType[], this.entries[entryUid]);
             }
 
             this.lookForReference([{ locale: code, uid, name: title }], ctSchema, this.entries[entryUid]);
+
+            const fields = this.missingMandatoryFields[uid];
+            const isPublished = entry.publish_details.length > 0;
+            if ((this.fix && fields && isPublished) || (!this.fix && fields)) {
+              const fixStatus = this.fix ? 'Fixed' : '';
+              fields?.forEach((field: { isPublished: boolean; fixStatus?: string }) => {
+                field.isPublished = isPublished;
+                if (this.fix && isPublished) {
+                  field.fixStatus = fixStatus;
+                }
+              });
+
+              if (this.fix && isPublished) {
+                this.log($t(auditFixMsg.ENTRY_MANDATORY_FIELD_FIX, { uid, locale: code }), 'error');
+                entry.publish_details = [];
+              }
+            } else {
+              delete this.missingMandatoryFields[uid];
+            }
+
             const message = $t(auditMsg.SCAN_ENTRY_SUCCESS_MSG, {
               title,
               local: code,
@@ -132,7 +157,12 @@ export default class Entries {
     // this.log('', 'info'); // Adding empty line
 
     this.removeEmptyVal();
-    return { missingEntryRefs: this.missingRefs, missingSelectFeild: this.missingSelectFeild };
+
+    return {
+      missingEntryRefs: this.missingRefs,
+      missingSelectFeild: this.missingSelectFeild,
+      missingMandatoryFields: this.missingMandatoryFields,
+    };
   }
 
   /**
@@ -149,6 +179,11 @@ export default class Entries {
         delete this.missingSelectFeild[propName];
       }
     }
+    for (let propName in this.missingMandatoryFields) {
+      if (!this.missingMandatoryFields[propName].length) {
+        delete this.missingMandatoryFields[propName];
+      }
+    }
   }
 
   /**
@@ -237,7 +272,16 @@ export default class Entries {
 
     for (const child of field?.schema ?? []) {
       const { uid } = child;
-      if (!entry?.[uid]) continue;
+      this.missingMandatoryFields[this.currentUid].push(
+        ...this.validateMandatoryFields(
+          [...tree, { uid: field.uid, name: child.display_name, field: uid }],
+          child,
+          entry,
+        ),
+      );
+      if (!entry?.[uid] && !child.hasOwnProperty('display_type')) {
+        continue;
+      }
 
       switch (child.data_type) {
         case 'reference':
@@ -668,7 +712,24 @@ export default class Entries {
    */
   validateSelectField(tree: Record<string, unknown>[], fieldStructure: SelectFeildStruct, field: any) {
     const { display_name, enum: selectOptions, multiple, min_instance, display_type } = fieldStructure;
-
+    if (field === null || field === '' || field?.length === 0 || !field) {
+      let missingCTSelectFieldValues = 'Not Selected';
+      return [
+        {
+          uid: this.currentUid,
+          name: this.currentTitle,
+          display_name,
+          display_type,
+          missingCTSelectFieldValues,
+          min_instance: min_instance ?? 'NA',
+          tree,
+          treeStr: tree
+            .map(({ name }) => name)
+            .filter((val) => val)
+            .join(' ➜ '),
+        },
+      ];
+    }
     let missingCTSelectFieldValues;
 
     if (multiple) {
@@ -714,7 +775,7 @@ export default class Entries {
    * @returns
    */
   fixSelectField(tree: Record<string, unknown>[], field: SelectFeildStruct, entry: any) {
-    const { enum: selectOptions, multiple, min_instance, display_type, display_name } = field;
+    const { enum: selectOptions, multiple, min_instance, display_type, display_name, uid } = field;
 
     let missingCTSelectFieldValues;
     let isMissingValuePresent = false;
@@ -724,7 +785,7 @@ export default class Entries {
       let { notPresent, filteredFeild } = obj;
       entry = filteredFeild;
       missingCTSelectFieldValues = notPresent;
-      if(missingCTSelectFieldValues.length) {
+      if (missingCTSelectFieldValues.length) {
         isMissingValuePresent = true;
       }
       if (min_instance && Array.isArray(entry)) {
@@ -736,20 +797,24 @@ export default class Entries {
             .slice(0, missingInstances)
             .map((choice) => choice.value);
           entry.push(...newValues);
+          this.log($t(auditFixMsg.ENTRY_SELECT_FIELD_FIX, { value: newValues.join(' '), uid }), 'error');
         }
       } else {
         if (entry.length === 0) {
           isMissingValuePresent = true;
           const defaultValue = selectOptions.choices.length > 0 ? selectOptions.choices[0].value : null;
           entry.push(defaultValue);
+          this.log($t(auditFixMsg.ENTRY_SELECT_FIELD_FIX, { value: defaultValue as string, uid }), 'error');
         }
       }
     } else {
       const isPresent = selectOptions.choices.some((choice) => choice.value === entry);
       if (!isPresent) {
         missingCTSelectFieldValues = entry;
         isMissingValuePresent = true;
-        entry = selectOptions.choices.length > 0 ? selectOptions.choices[0].value : null;
+        let defaultValue = selectOptions.choices.length > 0 ? selectOptions.choices[0].value : null;
+        entry = defaultValue;
+        this.log($t(auditFixMsg.ENTRY_SELECT_FIELD_FIX, { value: defaultValue as string, uid }), 'error');
       }
     }
     if (display_type && isMissingValuePresent) {
@@ -770,6 +835,46 @@ export default class Entries {
     }
     return entry;
   }
+
+  validateMandatoryFields(tree: Record<string, unknown>[], fieldStructure: any, entry: any) {
+    const { display_name, multiple, data_type, mandatory, field_metadata, uid } = fieldStructure;
+
+    const isJsonRteEmpty = () => {
+      const jsonNode = multiple
+        ? entry[uid]?.[0]?.children?.[0]?.children?.[0]?.text
+        : entry[uid]?.children?.[0]?.children?.[0]?.text;
+      return jsonNode === '';
+    };
+
+    const isEntryEmpty = () => {
+      const fieldValue = multiple ? entry[uid]?.length : entry[uid];
+      return fieldValue === '' || !fieldValue;
+    };
+
+    if (mandatory) {
+      if (
+        (data_type === 'json' && field_metadata.allow_json_rte && isJsonRteEmpty()) ||
+        (!(data_type === 'json') && isEntryEmpty())
+      ) {
+        return [
+          {
+            uid: this.currentUid,
+            name: this.currentTitle,
+            display_name,
+            missingFieldUid: uid,
+            tree,
+            treeStr: tree
+              .filter(({ name }) => name)
+              .map(({ name }) => name)
+              .join(' ➜ '),
+          },
+        ];
+      }
+    }
+
+    return [];
+  }
+
   /**
    *
    * @param field It contains the value to be searched
@@ -779,10 +884,10 @@ export default class Entries {
   findNotPresentSelectField(field: any, selectOptions: any) {
     let present = [];
     let notPresent = [];
-    const choicesMap = new Map(selectOptions.choices.map((choice: { value: any; }) => [choice.value, choice]));
+    const choicesMap = new Map(selectOptions.choices.map((choice: { value: any }) => [choice.value, choice]));
     for (const value of field) {
-      const choice:any = choicesMap.get(value);
-    
+      const choice: any = choicesMap.get(value);
+
       if (choice) {
         present.push(choice.value);
       } else {

packages/contentstack-audit/src/types/content-types.ts

@@ -16,6 +16,7 @@ type ContentTypeStruct = {
   title: string;
   description: string;
   schema?: ContentTypeSchemaType[];
+  mandatory: boolean;
 };
 
 type ModuleConstructorParam = {
@@ -38,6 +39,7 @@ type CommonDataTypeStruct = {
     ref_multiple: boolean;
     allow_json_rte: boolean;
   } & AnyProperty;
+  mandatory: boolean
 };
 
 type RefErrorReturnType = {
@@ -138,7 +140,9 @@ enum OutputColumn {
   'missingCts' = 'content_types',
   'Missing Branches' = 'branches',
   'MissingValues' = 'missingCTSelectFieldValues',
-  "Minimum Required Instaces" = 'min_instance'
+  'Minimum Required Instaces' = 'min_instance',
+  'missingFieldUid' = 'missingFieldUid',
+  'isPublished' = 'isPublished'
 }
 
 export {

packages/contentstack-audit/src/types/entries.ts

@@ -10,6 +10,7 @@ type Locale = {
 type EntryStruct = {
   uid: string;
   title: string;
+  publish_details: []
 } & {
   [key: string]:
     | EntryReferenceFieldDataType[]