* 2019-10-24 (**Version 4.3.0**)

dynamicdan · dynamicdan · commit b1fd830f7ff9 · 2019-10-24T10:53:51.000+02:00
* New features/support to help with downloading the **entire** (!) instance setup
   * added new search option templates and helpful notes for day-2-day search/resync workflows
   * added more system type tables like choice, props, dictionary etc. to extract more config
   * allow searching without the sys_class_name restriction to include all records from child tables or when that table doesn't have the field
     * Provide `"noClassName": true` in folder config per desired folder
     * See also the readme section for "Folder definitions"
 * Support for mass API calls to an instance to avoid HTTP 427 responses (better `--resync` support)
diff --git a/CHANGES.md b/CHANGES.md
@@ -1,5 +1,15 @@
 # FileSync Changes
 
+* 2019-10-24 (**Version 4.3.0**)
+ * New features/support to help with downloading the **entire** (!) instance setup
+   * added new search option templates and helpful notes for day-2-day search/resync workflows
+   * added more system type tables like choice, props, dictionary etc. to extract more config
+   * allow searching without the sys_class_name restriction to include all records from child tables or when that table doesn't have the field
+     * Provide `"noClassName": true` in folder config per desired folder
+     * See also the readme section for "Folder definitions"
+ * Support for mass API calls to an instance to avoid HTTP 427 responses (better `--resync` support)
+
+
 * 2017-03-26 (**Version 4.2.4**)
  * Add debug option via the command line additional to config setting. (`--debug`)
  * Fix export not using subdir format
diff --git a/README.md b/README.md
@@ -1,7 +1,7 @@
-sn-filesync -- ServiceNow FileSync (v4.2.4)
+sn-filesync -- ServiceNow FileSync
 =================
 
-[![NPM](https://nodei.co/npm-dl/sn-filesync.png?height=3&months=9)](https://nodei.co/npm-dl/sn-filesync/)
+[![NPM](https://nodei.co/npm/sn-filesync.png?downloadRank=true&stars=true)](https://nodei.co/npm-dl/sn-filesync/)
 
 [![Intro to FileSync](https://raw.githubusercontent.com/dynamicdan/filesync/master/thumbnail.png)](https://www.youtube.com/watch?v=OlVllfPVOrA "Intro to FileSync")
 
@@ -219,7 +219,14 @@ See the **lib/records.config.json** file for sample definitions.
                 "client.js": "client_script",    //   for ui pages, you might have three separate files:
                 "server.js": "processing_script" //    mypage.xhtml, mypage.client.js, mypage.server.js
             }                                    //   to store the all script associated with the page
-        }
+        },
+        "sys_choice": {
+            "_info": "Choices",
+            "table": "sys_choice",
+            "key": "element",
+            "subDirPattern": "name",
+            "noClassName": true                 // allow searching all child tables of sys_choice OR for when sys_class_name isn't present (!) on the table but it's used in a search query
+        },
         ...
     },
 ```
diff --git a/app.config.json b/app.config.json
@@ -21,11 +21,46 @@
     },
 
     "search": {
+
+
+         "__notes_daily_search": {
+            "readme": "Command line searches to do prior to development start. Full download of all the relevant/new stuff which can be completed as the first step to get the latest version of all config. WARNING: do a sanity check to verify credentials are still valid with limit of 1 first to avoid potential account lock out! (--records_per_search 1)",
+
+            "optional_step0_sanity_check": "./bin/app.js --config ../some/folder/app.config.json --search mine --records_per_search 1",
+
+            "step1": "./bin/app.js --config ../some/folder/app.config.json --search mine --download",
+
+            "step2": "./bin/app.js --config ../some/folder/app.config.json --search team-but-me --download",
+
+            "then_start_work": "./bin/app.js --config ../some/folder/app.config.json"
+        },
+
+
+
+        "all-from2019": {
+            "query": "sys_created_on>javascript:gs.dateGenerate('2018-12-31','23:59:59')",
+            "records_per_search": "10",
+            "download": false
+        },
+
+        "init": {
+            "_note": "Query will run across all defined tables. Somewhat experimental in terms of mass download. Once downloaded, will work fine.",
+            "query": "ORDERBYDESCsys_updated_on",
+            "records_per_search": "10000",
+            "download": false
+        },
+
         "mine": {
-            "query": "active=true^sys_updated_by=javascript:gs.getUserName()^ORsys_created_by=javascript:gs.getUserName()^ORDERBYDESCsys_updated_on",
-            "records_per_search": "100"
+            "query": "sys_updated_by=javascript:gs.getUserName()^ORsys_created_by=javascript:gs.getUserName()^ORDERBYDESCsys_updated_on",
+            "records_per_search": "300"
         },
-        "team": {
+        "team-but-me": {
+            "_note": "change the date to reflect when your team/instance was first setup",
+            "query": "sys_created_on>javascript:gs.dateGenerate('2019-06-01','00:00:00')^sys_created_by!=javascript:gs.getUserName()^sys_created_by!=admin^ORDERBYDESCsys_updated_on",
+            "records_per_search": "300"
+        },
+        "team_active": {
+            "_note": "change the date to reflect when your team/instance was first setup",
             "query": "active=true^sys_created_on>javascript:gs.dateGenerate('2015-03-25','23:59:59')^sys_created_by!=javascript:gs.getUserName()^sys_updated_by!=javascript:gs.getUserName()^sys_created_by!=admin^ORDERBYDESCsys_updated_on",
             "records_per_search": "100"
         },
@@ -41,7 +76,71 @@
             "records_per_search": "1",
             "download": true,
             "full_record": true
+        },
+
+
+
+        "__notes_instance_download": {
+            "info": "The below config allows downloading the entier data model including fields and labels. The following ~90K column/dictionary entries are broken down into groups under 10K to avoid exceeding the 10K query limit. The complete cmdb is ignored (~57K records)."
+        },
+        "t1": {
+            "table": "sys_dictionary",
+            "query": "nameSTARTSWITHa^ORnameSTARTSWITHb^ORnameSTARTSWITHc^ORnameSTARTSWITHd^ORnameSTARTSWITHe^ORnameSTARTSWITHf^ORnameSTARTSWITHg^ORnameSTARTSWITHh^ORnameSTARTSWITHi^ORnameSTARTSWITHj^ORnameSTARTSWITHk^ORnameSTARTSWITHl^ORnameSTARTSWITHm^ORnameSTARTSWITHn^nameNOT LIKEcmdb_",
+            "records_per_search": "10000",
+            "download": true,
+            "full_record": true
+        },
+        "t2": {
+            "table": "sys_dictionary",
+            "query": "nameSTARTSWITHo^ORnameSTARTSWITHp^ORnameSTARTSWITHq^ORnameSTARTSWITHr^ORnameSTARTSWITHt^ORnameSTARTSWITHu^ORnameSTARTSWITHv^ORnameSTARTSWITHw^ORnameSTARTSWITHx^ORnameSTARTSWITHy^ORnameSTARTSWITHz^nameNOT LIKEcmdb_",
+            "records_per_search": "10000",
+            "download": true,
+            "full_record": true
+        },
+        "t3": {
+            "table": "sys_dictionary",
+            "query": "nameSTARTSWITHs^nameNOT LIKEcmdb_^nameNOT LIKEsys_",
+            "records_per_search": "10000",
+            "download": true,
+            "full_record": true
+        },
+        "t4": {
+            "table": "sys_dictionary",
+            "query": "nameSTARTSWITHs^nameNOT LIKEcmdb_^nameLIKEsys_",
+            "records_per_search": "10000",
+            "download": true,
+            "full_record": true
+        },
+        "d1": {
+            "table": "sys_documentation",
+            "query": "nameSTARTSWITHa^ORnameSTARTSWITHb^ORnameSTARTSWITHc^ORnameSTARTSWITHd^ORnameSTARTSWITHe^ORnameSTARTSWITHf^ORnameSTARTSWITHg^ORnameSTARTSWITHh^ORnameSTARTSWITHi^ORnameSTARTSWITHj^ORnameSTARTSWITHk^ORnameSTARTSWITHl^ORnameSTARTSWITHm^ORnameSTARTSWITHn^nameNOT LIKEcmdb_",
+            "records_per_search": "10000",
+            "download": true,
+            "full_record": true
+        },
+        "d2": {
+            "table": "sys_documentation",
+            "query": "nameSTARTSWITHo^ORnameSTARTSWITHp^ORnameSTARTSWITHq^ORnameSTARTSWITHr^ORnameSTARTSWITHt^ORnameSTARTSWITHu^ORnameSTARTSWITHv^ORnameSTARTSWITHw^ORnameSTARTSWITHx^ORnameSTARTSWITHy^ORnameSTARTSWITHz^nameNOT LIKEcmdb_",
+            "records_per_search": "10000",
+            "download": true,
+            "full_record": true
+        },
+        "d3": {
+            "table": "sys_documentation",
+            "query": "nameSTARTSWITHs^nameNOT LIKEcmdb_^nameNOT LIKEsys_",
+            "records_per_search": "10000",
+            "download": true,
+            "full_record": true
+        },
+        "d4": {
+            "table": "sys_documentation",
+            "query": "nameSTARTSWITHs^nameNOT LIKEcmdb_^nameLIKEsys_",
+            "records_per_search": "10000",
+            "download": true,
+            "full_record": true
         }
+
+
     },
 
     "preLoad": true,
diff --git a/bin/app.js b/bin/app.js
@@ -10,8 +10,7 @@ var chokidar = require('chokidar');
 require('colors');
 var fs = require('fs-extra');
 var path = require('path');
-// used to generate a hash of a file
-var crypto = require('crypto');
+
 var glob = require("glob");
 var winston = require('winston');
 var moment = require('moment');
@@ -49,7 +48,10 @@ var constants = {
     DOWNLOAD_OK: 1,
     DOWNLOAD_FAIL: -1,
 
-    SLASH: '/'
+    SLASH: '/',
+
+    // max number of requests allowed to be running at once
+    instanceAPITolerance: 30
 
     //isMac: /^darwin/.test(process.platform)
     //isWin = /^win/.test(process.platform)
@@ -828,6 +830,7 @@ function addToPreLoadList(filePath, options) {
     }
 }
 
+// TODO: not currently used... is this a bad thing?
 function addIfNotPresent(filePath) {
     fs.exists(filePath, function (exists) {
         if (!exists) {
@@ -1041,6 +1044,17 @@ function receive(file, allDoneCallBack) {
         // this is a combination of sys_id and name. Use sys_id from name when possible and ignore the query
         query = sys_id ? '' : map.key + '=' + map.keyValue;
 
+
+    // avoid overwhleming the instance and getting HTTP 427 responses
+    if(filesInQueueToDownload >= constants.instanceAPITolerance) {
+        setTimeout(function() {
+            receive(file, allDoneCallBack);
+        }, 200);
+
+        logit.debug('Delaying request to avoid HTTP 427 errors.');
+        return;
+    }
+
     logit.debug('Adding:', {
         file: file,
         table: map.table,
diff --git a/lib/file-record.js b/lib/file-record.js
@@ -28,7 +28,18 @@ function makeHash(data) {
     return hash1;
 }
 
-function getFieldMap(filename, map) {
+method.getFieldMap = function(filename, map) {
+
+    if(typeof map.fields == 'undefined') {
+        // if this is the case then the config doesn't specify which fields should be downloaded for the record.
+        // this means that we are only interested in the full record
+        // todo: support resyncing the full record
+        this.logger.debug("No fields defined for this file type");
+        this.logger.debug('map...');
+        this.logger.debug(map);
+        return null;
+    }
+
     var suffixes = Object.keys(map.fields);
 
     // sort suffixes so most specific suffixes are looped through first...
@@ -51,7 +62,7 @@ function getFieldMap(filename, map) {
         }
     }
     return null;
-}
+};
 
 // fix windows path issues (windows can handle both '\' and '/' so be *nix friendly)
 function normalisePath(p) {
@@ -245,7 +256,7 @@ method.getSyncMap = function () {
     }
 
     // validate file suffix is mapped
-    var fieldMap = getFieldMap(fileName, map);
+    var fieldMap = this.getFieldMap(fileName, map);
     if (!fieldMap) {
         // note that full_record download will not have a mapped suffix
         this.logger.warn('No field map fileName: %s',fileName);
diff --git a/lib/records.config.json b/lib/records.config.json
@@ -51,7 +51,7 @@
             "fields": {
                 "js": "script"
             },
-            "subDirPattern": "client_<client_callable>"
+            "subDirPattern": "scope_<sys_scope>/client_<client_callable>"
         },
         "style_sheets": {
             "table": "content_css",
@@ -185,6 +185,91 @@
                 "html": "template"
             },
             "subDirPattern": "sp_widget"
+        },
+
+
+
+
+
+
+
+
+        "sys_properties": {
+            "table": "sys_properties",
+            "key": "name",
+            "subDirPattern": "<sys_scope>",
+            "_custom": true
+        },
+        "sys_app_module": {
+            "_info": "Nav modules",
+            "table": "sys_app_module",
+            "key": "title",
+            "subDirPattern": "application",
+            "searchConstraint": "titleISNOTEMPTY",
+            "_info_searchConstraint": "Avoid separators",
+            "_custom": true
+        },
+        "sys_data_policy2": {
+            "table": "sys_data_policy2",
+            "key": "model_table",
+            "subDirPattern": "sys_package",
+            "_custom": true
+        },
+        "sysevent_register": {
+            "table": "sysevent_register",
+            "key": "event_name",
+            "subDirPattern": "sys_scope/table",
+            "_custom": true
+        },
+        "sys_choice": {
+            "_info": "Choices",
+            "table": "sys_choice",
+            "key": "element",
+            "subDirPattern": "name",
+            "noClassName": true,
+            "_custom": true
+        },
+        "tables": {
+            "_info": "Tables",
+            "table": "sys_db_object",
+            "key": "name",
+            "subDirPattern": "sys_package",
+            "searchConstraint": "sys_nameISNOTEMPTY",
+            "_custom": true
+        },
+        "sys_dictionary": {
+            "_info": "fields",
+            "table": "sys_dictionary",
+            "key": "element",
+            "subDirPattern": "sys_scope/name",
+            "_note": "name = table system name. sys_ fields have no scope and we do not want them.",
+            "searchConstraint": "elementISNOTEMPTY^sys_scopeISNOTEMPTY",
+            "_custom": true
+        },
+        "sys_documentation": {
+            "_info": "Field labels",
+            "table": "sys_documentation",
+            "key": "element",
+            "subDirPattern": "sys_scope/name",
+            "searchConstraint": "language=en^sys_scopeISNOTEMPTY",
+            "_custom": true
+        },
+        "sys_user_role": {
+            "_info": "user roles",
+            "table": "sys_user_role",
+            "key": "name",
+            "subDirPattern": "sys_package",
+            "_custom": true
+        },
+        "sys_security_acl": {
+            "_info": "ACLs",
+            "table": "sys_security_acl",
+            "key": "sys_name",
+            "fields": {
+                "js": "script"
+            },
+            "subDirPattern": "type/sys_name/operation",
+            "_custom": true
         }
     }
 }
diff --git a/lib/search.js b/lib/search.js
@@ -116,6 +116,15 @@ method.getResults = function (queryObj, callback) {
         db.folder = folder;
         db.table = folObj.table;
         db.subDir = folObj.subDirPattern || false;
+
+        // if a table has a search constraint defined then AND it to the existing query
+        if(folObj.searchConstraint) {
+            db.query += '^' + folObj.searchConstraint;
+        }
+
+        // overrides the default search logic to restrict search by the table sys_class_name
+        db.noClassName = folObj.noClassName ? true : false;
+
         // fields are optional
         db.fields = queryObj.recordOnly ? false : folObj.fields;
 
diff --git a/lib/snc-client.js b/lib/snc-client.js
@@ -175,8 +175,12 @@ var sncClient = restler.service(function sncClient(config) {
             };
             if (query.query) {
                 parms.parmValue = query.query;
-                // ensures that tables that are extended are still restricted to 1 table
-                parms.parmValue += "^sys_class_name=" + tableName;
+                if(query.noClassName) {
+                    // don't add the additional query constraint
+                } else {
+                    // ensures that tables that are extended are still restricted to 1 table
+                    parms.parmValue += "^sys_class_name=" + tableName;
+                }
             }
             if (query.rows) {
                 parms.rows = query.rows;
diff --git a/lib/string-util.js b/lib/string-util.js
@@ -76,7 +76,7 @@ method.extractTableFromName = function(str) {
          * Example
          * sp_widget_c6545050ff223100ba13ffffffffffe8
          */
-        var sys_id = StringUtil.extractSysIdFromName(str);
+        var sys_id = method.extractSysIdFromName(str);
         table = str.replace('_' + sys_id, '');
     }
 

Original file line number	Diff line number	Diff line change
`@@ -76,7 +76,7 @@ method.extractTableFromName = function(str) {`
`76`	`76`	`* Example`
`77`	`77`	`* sp_widget_c6545050ff223100ba13ffffffffffe8`
`78`	`78`	`*/`
`79`		`- var sys_id = StringUtil.extractSysIdFromName(str);`
	`79`	`+ var sys_id = method.extractSysIdFromName(str);`
`80`	`80`	`table = str.replace('_' + sys_id, '');`
`81`	`81`	`}`
`82`	`82`