Add support for policy useBf option

BillysCoolJob · Jun 11, 2024 · 1f4ed58 · 1f4ed58
1 parent a467f14
commit 1f4ed58
Show file tree

Hide file tree

Showing 13 changed files with 342 additions and 63 deletions.
diff --git a/alarm/PolicyManager2.js b/alarm/PolicyManager2.js
@@ -1530,9 +1530,19 @@ class PolicyManager2 {
               wanUUID,
               routeType
             });
+            if (policy.useBf) {
+              await domainBlock.blockCategory(target + "_bf", {pid,
+                scope: scope, category: target + "_bf", intfs, guids,
+                action: action, tags, parentRgId, seq, wanUUID, routeType, append: true
+              });
+            }
           }
         }
+
         await categoryUpdater.activateCategory(target);
+        if (policy.useBf) {
+          await categoryUpdater.activateCategory(target+'_bf', true);
+        }
         if (action === "allow") {
           remoteSet4 = categoryUpdater.getAllowIPSetName(target);
           remoteSet6 = categoryUpdater.getAllowIPSetNameForIPV6(target);

diff --git a/control/CategoryUpdater.js b/control/CategoryUpdater.js
@@ -309,13 +309,17 @@ class CategoryUpdater extends CategoryUpdaterBase {
     return this.customizedCategories;
   }
 
-  async activateCategory(category) {
-    if (this.isActivated(category)) return;
-    if (firewalla.isMain()) // do not create ipset unless in FireMain
+  async activateCategory(category, reloadFromCloud=false) {
+    log.debug("invoke activate category", category, ", reloadFromCloud", reloadFromCloud)
+    if (this.isActivated(category) && !reloadFromCloud) return;
+    if (firewalla.isMain() && !reloadFromCloud) // do not create ipset unless in FireMain
       await super.activateCategory(category, this.isCustomizedCategory(category) ? this._getCustomizedCategoryIpsetType(category) : "hash:net");
+    else
+      this.activeCategories[category] = 1
     sem.emitEvent({
       type: "Policy:CategoryActivated",
       toProcess: "FireMain",
+      reloadFromCloud: reloadFromCloud,
       message: "Category activated: " + category,
       category: category
     });
@@ -1245,7 +1249,7 @@ class CategoryUpdater extends CategoryUpdaterBase {
 
   // system target list using cloudcache, mainly for large target list to reduce bandwidth usage of polling hashset
   isManagedTargetList(category) {
-    return !this.isUserTargetList(category) && !this.isSmallExtendedTargetList(category) && !this.excludeListBundleIds.has(category);
+    return !this.isUserTargetList(category) && !this.isSmallExtendedTargetList(category) && !this.excludeListBundleIds.has(category) && !category.endsWith('_bf');
   }
 }
 

diff --git a/extension/dnsmasq/dnsmasq.js b/extension/dnsmasq/dnsmasq.js
@@ -704,7 +704,7 @@ module.exports = class DNSMASQ {
             }
           }
           const filePath = `${FILTER_DIR}/${name}.conf`;
-          await fs.writeFileAsync(filePath, entries.join('\n'));
+          await this.writeFileAsync(filePath, entries.join('\n'), options && options.append);
         }
 
         if (!_.isEmpty(options.intfs)) {
@@ -727,7 +727,7 @@ module.exports = class DNSMASQ {
               }
             }
             const filePath = `${NetworkProfile.getDnsmasqConfigDirectory(intf)}/${name}.conf`;
-            await fs.writeFileAsync(filePath, entries.join('\n'));
+            await this.writeFileAsync(filePath, entries.join('\n'), options && options.append);
           }
         }
 
@@ -750,7 +750,7 @@ module.exports = class DNSMASQ {
               }
             }
             const filePath = `${FILTER_DIR}/tag_${tag}_${name}.conf`;
-            await fs.writeFileAsync(filePath, entries.join('\n'));
+            await this.writeFileAsync(filePath, entries.join('\n'), options && options.append);
           }
         }
 
@@ -777,7 +777,7 @@ module.exports = class DNSMASQ {
                   break;
                 }
               }
-              await fs.writeFileAsync(filePath, entries.join('\n'));
+              await this.writeFileAsync(filePath, entries.join('\n'), options && options.append);
             }
           }
         }
@@ -808,13 +808,20 @@ module.exports = class DNSMASQ {
           }
         }
         const filePath = `${FILTER_DIR}/${name}.conf`;
-        await fs.writeFileAsync(filePath, entries.join('\n'));
+        await this.writeFileAsync(filePath, entries.join('\n'), options && options.append);
       }
     }).catch((err) => {
       log.error("Failed to add category mac set entry into file:", err);
     });
   }
 
+  async writeFileAsync(filePath, content, append=false) {
+    if (append) {
+      return await fs.appendFileAsync(filePath, "\n" + content);
+    }
+    return await fs.writeFileAsync(filePath, content);
+  }
+
   async writeAllocationOption(tagName, policy, known = false) {
     await lock.acquire(LOCK_OPS, async () => {
       log.info('Writting allocation file for', tagName)

diff --git a/intel/SSLIntel.js b/intel/SSLIntel.js
@@ -27,7 +27,7 @@ class SSLIntel extends Intel {
     if (ip) {
       try {
         let sslInfo = await intelTool.getSSLCertificate(ip);
-        log.info("Get ssl info of " + ip + ": " + sslInfo);
+        log.info("Get ssl info of " + ip + ": " + JSON.stringify(sslInfo));
         if (sslInfo) {
           for (var key in sslInfo) {
             let detailKey = "e.dest.ssl." + key;

diff --git a/lib/Bone.js b/lib/Bone.js
@@ -17,6 +17,7 @@
 const firewalla = require('../net2/Firewalla.js');
 const log = require("../net2/logger.js")(__filename);
 
+const fc = require('../net2/config.js');
 const config = require('../net2/config.js').getConfig();
 
 // sometimes timeout could take 2 mins
@@ -450,7 +451,7 @@ exports.intelAsync = async function (ip, action, intel) {
 
   boneAPITimeSeries.recordHit('intel').exec()
 
-  const response = await rrWithErrHandling(options, true);
+  const response = await rrWithErrHandling(options, true && !fc.getConfig().disableHttpsPool);
 
   if (firewalla.isMain() && response && response.headers) {
     rateLimit.recordRate(response.headers);

diff --git a/net2/config.js b/net2/config.js
@@ -178,7 +178,7 @@ async function reloadConfig() {
     let testConfigFile = f.getUserConfigFolder() + "/config.test.json";
     // will throw error if not exist
     await fs.promises.access(testConfigFile, fs.constants.F_OK | fs.constants.R_OK)
-    testConfig = JSON.parse(await fs.promises.readFile(userConfigFile, 'utf8'))
+    testConfig = JSON.parse(await fs.promises.readFile(testConfigFile, 'utf8'))
     log.warn("Test config is being used", testConfig);
   } catch(err) {
     // clears config on any error

diff --git a/sensor/CategoryExaminerPlugin.js b/sensor/CategoryExaminerPlugin.js
@@ -15,6 +15,7 @@
 
 const log = require('../net2/logger.js')(__filename);
 
+const _ = require('lodash');
 const LRU = require('lru-cache');
 const sl = require('./SensorLoader.js');
 const fc = require('../net2/config.js');
@@ -31,6 +32,8 @@ const categoryFastFilterFeature = "category_filter";
 
 const scheduler = require('../util/scheduler');
 const firewalla = require('../net2/Firewalla.js');
+const IntelTool = require('../net2/IntelTool.js');
+const intelTool = new IntelTool();
 
 const sclient = require('../util/redis_manager.js').getSubscriptionClient();
 const Hashes = require('../util/Hashes');
@@ -67,6 +70,7 @@ class CategoryExaminerPlugin extends Sensor {
       sclient.on("message", async (channel, message) => {
         switch (channel) {
           case BF_SERVER_MATCH: {
+            log.debug("receive message from", channel, message)
             let msgObj;
             try {
               msgObj = JSON.parse(message);
@@ -240,7 +244,7 @@ class CategoryExaminerPlugin extends Sensor {
     try {
       response = await this.matchDomain(origDomain);
     } catch (e) {
-      log.debug(`Fail to get match result from category filter: ${origDomain}`);
+      log.warn(`Fail to get match result from category filter: ${origDomain}`);
       return;
     }
 
@@ -250,6 +254,7 @@ class CategoryExaminerPlugin extends Sensor {
       const matchedDomain = result.item;
       if (!categoryUpdater.isActivated(category)) {
         // do not check if category is not activated
+        log.info("skip update domain of disabled category", category);
         continue;
       }
       if (status === "Match") {
@@ -271,7 +276,19 @@ class CategoryExaminerPlugin extends Sensor {
     }
   }
 
+  async _findCategory(domain) {
+    const resp = await intelTool.checkIntelFromCloud(null, domain);
+    if (!_.isArray(resp) || resp.length == 0) {
+      return null;
+    }
+    return _.uniq(resp.map((i) => {if (i.c == 'ad') {return 'adblock_strict';} else return i.c}));
+  }
+
   async confirmJob() {
+    if (this.confirmSet.size == 0) {
+      return;
+    }
+    log.info("category examiner run comfirm job", JSON.stringify(JSON.stringify([...this.confirmSet])));
     const categoryDomainMap = new Map();
     for (const item of this.confirmSet) {
       const [category, matchedDomain, origDomain] = item.split(":");
@@ -283,32 +300,29 @@ class CategoryExaminerPlugin extends Sensor {
     }
 
     this.confirmSet = new Set();
-
     for (const [category, domainList] of categoryDomainMap) {
       const strategy = await categoryUpdater.getStrategy(category);
-      const matchedDomainList = domainList.map(item => item[0]); // send matched domains from local bf to cloud for confirmation, original domain can be a subdomain of matched domain
-
       // update hit set using matched domain list
       if (strategy.updateConfirmSet) {
         let score = Date.now();
-        let results = await this.confirmDomains(category, strategy, matchedDomainList);
-        if (results === null) {
-          log.debug("Fail to confirm domains of category", category);
-          return;
-        }
-        const positiveSet = new Set(results);
-        const unmatchedOrigDomainSet = new Set(domainList.map(item => item[1]));
-        for (const [matchedDomain, origDomain] of domainList) {
-          if (positiveSet.has(matchedDomain)) {
-            log.info(`Add domain ${matchedDomain} to hit set of ${category} `);
-            await this.addDomainToHitSet(category, matchedDomain, score);
+        let originCategory = category.split("_bf")[0];
+        const origDomainList = domainList.map(item => item[1]);
+        const unmatchedOrigDomainSet = new Set(origDomainList);
+        for (const origDomain of origDomainList) {
+          const categories = await this._findCategory(origDomain);
+          log.verbose("categories", category, categories, origDomain);
+          if (categories.includes(category) || categories.includes(originCategory)) { // matched with cloud data
+            log.info(`Add domain ${origDomain} to hit set of ${category} `);
+            await this.addDomainToHitSet(category, origDomain, score);
             unmatchedOrigDomainSet.delete(origDomain);
           }
         }
+
         for (const origDomain of unmatchedOrigDomainSet) {
           log.info(`Add domain ${origDomain} to passthrough set of ${category} `);
           await this.addDomainToPassthroughSet(category, origDomain, score);
         }
+
         await this.limitHitSet(category, MAX_CONFIRM_SET_SIZE);
         await this.limitPassthroughSet(category, MAX_CONFIRM_SET_SIZE);
 

diff --git a/sensor/CategoryUpdateSensor.js b/sensor/CategoryUpdateSensor.js
@@ -84,13 +84,21 @@ class CategoryUpdateSensor extends Sensor {
   resetCategoryHashsetMapping() {
     this.categoryHashsetMapping = {
       "games": "app.gaming",
+      "games_bf": "app.gaming_bf",
       "social": "app.social",
+      "social_bf": "app.social_bf",
       "av": "app.video",
+      "av_bf": "app.video_bf",
       "porn": "app.porn",  // dnsmasq redirect to blue hole if porn
+      "porn_bf": "app.porn_bf",
       "gamble": "app.gamble",
+      "gamble_bf": "app.gamble_bf",
       "shopping": "app.shopping",
+      "shopping_bf": "app.shopping_bf",
       "p2p": "app.p2p",
-      "vpn": "app.vpn"
+      "p2p_bf": "app.p2p_bf",
+      "vpn": "app.vpn",
+      "vpn_bf": "app.vpn_bf"
     }
   }
 
@@ -151,7 +159,7 @@ class CategoryUpdateSensor extends Sensor {
         await categoryUpdater.updateStrategy(category, "default");
       }
     } else {
-      await categoryUpdater.updateStrategy(category, "default");
+      await categoryUpdater.updateStrategy(category, !category.endsWith('_bf') ? "default": "filter");
     }
 
     let categoryStrategy = await categoryUpdater.getStrategy(category);