* Add debug option via the command line additional to config setting. (--debug)

 * Fix export not using subdir format
 * Add support for more tables (service portal related)
   * NOTE: Some folders like widgets are now sp_widgets (to group SP together). Ensure you rename both records/widgets and records/.sync_data/widgets to be "sp_widgets" OR define your own "folders" mapping in your config.json
 * Add more debugging for problematic folder/sync configs and failed downloads
 * Fix readme/documentation issues

Author: dynamicdan

CHANGES.md

@@ -1,7 +1,16 @@
 # FileSync Changes
 
+* 2017-03-26 (**Version 4.2.4**)
+ * Add debug option via the command line additional to config setting. (`--debug`)
+ * Fix export not using subdir format
+ * Add support for more tables (service portal related)
+   * NOTE: Some folders like widgets are now sp_widgets (to group SP together). Ensure you rename both records/widgets and records/.sync_data/widgets to be "sp_widgets" OR define your own "folders" mapping in your config.json
+ * Add more debugging for problematic folder/sync configs and failed downloads
+ * Fix readme/documentation issues
+
 * 2016-12-22 (**Version 4.2.3**)
- * Support searching for and downloading records via a URL as search criteria
+ * Improved support for searching and downloading records.
+   * Eg. `node bin/app.js --config config.json --search <URL>`
 
 * 2016-11-01 (**Version 4.2.2**)
  * Add new search feature allowing easier downloading of single records

README.md

@@ -1,4 +1,4 @@
-sn-filesync -- ServiceNow FileSync (v4.2.2)
+sn-filesync -- ServiceNow FileSync (v4.2.4)
 =================
 
 [![NPM](https://nodei.co/npm-dl/sn-filesync.png?height=3&months=9)](https://nodei.co/npm-dl/sn-filesync/)

bin/app.js

@@ -57,7 +57,9 @@ var constants = {
 constants.chokiWatcherIgnore = [/[\/\\]\./, '**/*' + constants.fullRecordSuffix];
 
 // default to ok.
-var multiDownloadStatus = constants.DOWNLOAD_OK;
+var multiDownloadStatus = constants.DOWNLOAD_OK,
+    // support the end user
+    listOfFailedFiles = [];
 
 var testsRunning = false;
 
@@ -98,6 +100,9 @@ function init() {
         winston.error('Configuration error:'.red, e.message);
         exitApp();
     }
+    if (argv.debug) {
+        config.debug = true;
+    }
 
     setupLogging();
 
@@ -253,9 +258,15 @@ function extractTableFromName(str) {
          * Example
          * http://localhost:16008/nav_to.do?uri=%2Fsp_widget.do%3Fsys_id%3Df37aa302cb70020000f8d856634c9cfc%26sysparm_record_target%3Dsp_widget
          *
+         * OR
+         *
+         * http://localhost:16008/nav_to.do?uri=sp_angular_provider.do?sys_id=06e836f0d722120023c84f80de6103a1"
+         *
          */
-        start = str.replace('_list', '').split('.do%3F');
-        end = start[0].split('uri=%2F');
+        // normalise the string
+        str = str.replace(/\%2F/g, '').replace(/\%3F/g, '');
+        start = str.replace('_list', '').split('.do');
+        end = start[1].split('uri=');
         table = end.pop();
 
     } else if (str.indexOf('.do') >= 0) {
@@ -881,6 +892,7 @@ function addConfigFiles() {
                 // each file to create
                 for (var file in root.preLoadList[folder]) {
                     var filePath = path.join(r, folder, root.preLoadList[folder][file]);
+
                     addToPreLoadList(filePath, {
                         filePath: filePath
                     });
@@ -897,6 +909,7 @@ function addToPreLoadList(filePath, options) {
         filePath: filePath
     };
     // only process if we don't already have it in the list
+    // todo: assumes all file names are unique!
     if (typeof filesToPreLoad[filePath] == 'undefined') {
         filesToPreLoad[filePath] = options;
         addFile(filePath);
@@ -934,7 +947,7 @@ function displayHelp() {
 function handleError(err, context) {
     logit.error(err);
     if (context) {
-        logit.error('  handleError context:', context);
+        logit.error('handleError context:', context);
     }
 }
 
@@ -970,9 +983,13 @@ function exportCurrentSetup(exportConfigPath) {
     for (var i in watchedFolders) {
         // remove sensitive data that may exist
         delete exportConfig.roots[watchedFolders[i]].auth;
+
+        // remove caching properties
+        delete exportConfig.roots[watchedFolders[i]].root;
+
         // overwrite remaining sensitive data
         exportConfig.roots[watchedFolders[i]].user = '<your user name>';
-        exportConfig.roots[watchedFolders[i]].pass = '<your password (which will be encrypted)>';
+        exportConfig.roots[watchedFolders[i]].pass = '<your password (which will become encrypted)>';
 
         exportConfig.roots[watchedFolders[i]].preLoadList = {};
     }
@@ -991,13 +1008,14 @@ function exportCurrentSetup(exportConfigPath) {
                 var f = new FileRecord(config, file),
                     folder = f.getFolderName(),
                     fileName = f.getFileName(),
+                    exportPath = f.getFileSubDirPath(),
                     rootDir = f.getRoot();
 
                 // add to appropriate preLoadList folder array
                 if (!exportConfig.roots[rootDir].preLoadList[folder]) {
                     exportConfig.roots[rootDir].preLoadList[folder] = [];
                 }
-                exportConfig.roots[rootDir].preLoadList[folder].push(fileName);
+                exportConfig.roots[rootDir].preLoadList[folder].push(exportPath);
             }
         })
         .on('ready', function () {
@@ -1050,7 +1068,11 @@ function decrementQueue() {
             notifyEnabled = true;
             // show one notification to represent if all files were downloaded or not
             if (multiDownloadStatus == constants.DOWNLOAD_FAIL) {
-                logit.error('Some or all files failed to download'.red);
+                logit.error('Files failed to download.'.red + ' See list below for details.');
+                // TODO, show what files failed
+                logit.info(listOfFailedFiles);
+                // reset list
+                listOfFailedFiles = [];
                 notifyUser(msgCodes.COMPLEX_ERROR);
             } else {
                 notifyUser(msgCodes.ALL_DOWNLOADS_COMPLETE);
@@ -1100,7 +1122,12 @@ function validResponse(err, obj, db, map, fileRecord) {
 
 function receive(file, allDoneCallBack) {
     var map = fileRecords[file].getSyncMap(),
-        fileMeta = fileRecords[file].getMeta();
+        fileMeta = fileRecords[file].getMeta(),
+        // the sys_id either comes from the meta (if already exists) or the file name or not at all
+        sys_id = fileMeta.sys_id || extractSysIdFromName(map.keyValue) || false,
+        // map.keyValue could be like "insert_problem.d5e561f3c0a8000901a883289848f88d"
+        // this is a combination of sys_id and name. Use sys_id from name when possible and ignore the query
+        query = sys_id ? '' : map.key + '=' + map.keyValue;
 
     logit.debug('Adding:', {
         file: file,
@@ -1117,12 +1144,13 @@ function receive(file, allDoneCallBack) {
     var db = {
         table: map.table,
         field: map.field,
-        query: map.key + '=' + map.keyValue,
-        sys_id: fileMeta.sys_id || false
+        query: query,
+        sys_id: sys_id
     };
 
     snc.table(db.table).getRecords(db, function (err, obj) {
         var isValid = validResponse(err, obj, db, map, fileRecords[file]);
+
         if (!isValid) {
             decrementQueue();
             allDoneCallBack(false);
@@ -1134,6 +1162,7 @@ function receive(file, allDoneCallBack) {
             objName = record.name; // TODO : use objName instead of file var.
 
 
+
         // legacy concept (still needed??... TODO: don't allow creation of 0 byte files!)
         if (record[db.field].length < 1) {
             logit.warn('**WARNING : this record is 0 bytes'.red);
@@ -1379,7 +1408,8 @@ function addFile(file, callback) {
     // default callback
     callback = callback || function (complete) {
         if (!complete) {
-            logit.info(('Could not add file:  ' + file).red);
+            logit.warn(('Could not add file:  ' + file));
+            listOfFailedFiles.push(file);
             multiDownloadStatus = constants.DOWNLOAD_FAIL;
         }
     };
@@ -1542,10 +1572,16 @@ function watchFolders() {
 
                         addFile(file);
                     }
+                } else {
+                    // file could be invalid (eg, folder mapping missing)
+                    logit.warn('File cannot be tracked because it is not valid: %s', file);
                 }
 
             } else {
-                trackFile(file);
+                if (!trackFile(file)) {
+                    // file could be invalid (eg, folder mapping missing)
+                    logit.warn('File cannot be tracked because it is not valid: %s', file);
+                }
             }
         })
         .on('change', onChange)

lib/file-record.js

@@ -106,11 +106,6 @@ method.clearMetaFile = function (callback) {
         }
     });
 };
-
-method.getFileName = function () {
-    return path.basename(this.filePath);
-};
-
 // track if a file has been discovered by the watcher or not as a "new" record field to download
 method.setNewlyDiscoveredFile = function (isNew) {
     this._isNewFile = isNew;
@@ -119,6 +114,22 @@ method.isNewlyDiscoveredFile = function () {
     return this._isNewFile;
 };
 
+/**
+ * Returns the full path to the file minus the path to the root dir
+ */
+method.getFilePathRelativeToRoot = function () {
+    return this.filePath.replace(this.rootDir + SLASH, '');
+};
+
+method.getFileSubDirPath = function () {
+    var p = this.getFilePathRelativeToRoot();
+    return p.replace(this.getFolderName() + SLASH, '');
+};
+
+method.getFileName = function () {
+    return path.basename(this.filePath);
+};
+
 method.getFolderName = function () {
     // remove file name from path
     var dir = path.dirname(this.filePath),
@@ -229,7 +240,7 @@ method.getSyncMap = function () {
     // validate parent folder is mapped
     var map = this.config.folders[folder];
     if (!map) {
-        this.logger.warn('No map');
+        this.logger.warn('No mapping for this folder: %s', folder);
         return null;
     }
 

lib/records.config.json

@@ -139,16 +139,37 @@
             },
             "subDirPattern": "web_service_definition"
         },
-        "widgets": {
+        "sp_widgets": {
             "table": "sp_widget",
             "key": "name",
             "fields": {
                 "client.js": "client_script",
                 "demo.js": "demo_data",
                 "server.js": "script",
                 "html": "template",
-                "scss": "scss"
-            }
+                "css": "css"
+            },
+            "subDirPattern": "servicenow_<servicenow>"
+        },
+        "sp_header_footer": {
+            "table": "sp_header_footer",
+            "key": "name",
+            "fields": {
+                "client.js": "client_script",
+                "demo.js": "demo_data",
+                "server.js": "script",
+                "html": "template",
+                "css": "css"
+            },
+            "subDirPattern": "servicenow_<servicenow>"
+        },
+        "sp_providers": {
+            "table": "sp_angular_provider",
+            "key": "name",
+            "fields": {
+                "js": "script"
+            },
+            "subDirPattern": "type"
         }
     }
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "sn-filesync",
-  "version": "4.2.3",
+  "version": "4.2.4",
   "description": "Enables out of instance record updates for ServiceNow.",
   "main": "bin/app.js",
   "bin": {