Refactor custom filters

extension-manifest-v3/src/background/adblocker.js

@@ -24,6 +24,7 @@ import { getMetadata } from '/utils/trackerdb.js';
 
 import { tabStats, updateTabStats } from './stats.js';
 import { getException } from './exceptions.js';
+import { customFiltersEngine } from './custom-filters.js';
 
 let enabledEngines = [];
 let options = {};
@@ -32,10 +33,12 @@ const regionalFiltersEngine = engines.init(engines.REGIONAL_ENGINE);
 
 const setup = asyncSetup([
   // Init engines
-  engines.init(engines.CUSTOM_ENGINE),
   engines.init(engines.FIXES_ENGINE),
   ENGINES.map(({ name }) => engines.init(name)),
-  // Regional filters engine is initialized separately for direct access
+
+  // Regional and custom filters engines are initialized separately
+  // for direct access
+  customFiltersEngine,
   regionalFiltersEngine,
 
   // Update options & enabled engines
@@ -44,8 +47,6 @@ const setup = asyncSetup([
 
     if (options.terms) {
       enabledEngines = [
-        // Add custom engine
-        engines.CUSTOM_ENGINE,
         engines.FIXES_ENGINE,
         // Main engines
         ...ENGINES.filter(({ key }) => options[key]).map(({ name }) => name),
@@ -54,6 +55,10 @@ const setup = asyncSetup([
       if (options.regionalFilters.enabled) {
         enabledEngines.push(engines.REGIONAL_ENGINE);
       }
+
+      if (options.customFilters.enabled) {
+        enabledEngines.push(engines.CUSTOM_ENGINE);
+      }
     } else {
       enabledEngines = [];
     }
@@ -72,19 +77,12 @@ const setup = asyncSetup([
 
   // Regional filters
   observe('regionalFilters', async ({ enabled, regions }, lastValue) => {
-    const engine = await regionalFiltersEngine;
-
-    if (
+    // Background startup
+    if (!lastValue) {
+      const engine = await regionalFiltersEngine;
       // Pre-requirement for skipping update - engine must be initialized
       // Otherwise it is a very first try to setup the engine
-      engine.lists.size &&
-      // 1. Background script startup
-      (!lastValue ||
-        // 2. Exact comparison of the values
-        (lastValue.enabled === enabled &&
-          lastValue.regions.join() === regions.join()))
-    ) {
-      return;
+      if (engine.lists.size) return;
     }
 
     // Clean previous regional engines

extension-manifest-v3/src/background/custom-filters.js

@@ -8,10 +8,108 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0
  */
-import { parseFilters } from '@cliqz/adblocker';
 
+import { store } from 'hybrids';
+import {
+  parseFilters,
+  detectFilterType,
+  FilterType,
+  CosmeticFilter,
+} from '@cliqz/adblocker';
+
+import {
+  createDocumentConverter,
+  createOffscreenConverter,
+} from '/utils/dnr-converter.js';
 import * as engines from '/utils/engines.js';
 
+import Options, { observe } from '/store/options.js';
+import CustomFilters from '/store/custom-filters.js';
+
+export const customFiltersEngine = engines.init(engines.CUSTOM_ENGINE);
+
+const convert =
+  __PLATFORM__ !== 'safari' && __PLATFORM__ !== 'firefox'
+    ? createOffscreenConverter()
+    : createDocumentConverter();
+
+class TrustedScriptletError extends Error {}
+
+// returns a scriptlet with encoded arguments
+// returns undefined if not a scriptlet
+// throws if scriptlet cannot be trusted
+function fixScriptlet(filter, trustedScriptlets) {
+  const cosmeticFilter = CosmeticFilter.parse(filter);
+
+  if (
+    !cosmeticFilter ||
+    !cosmeticFilter.isScriptInject() ||
+    !cosmeticFilter.selector
+  ) {
+    return null;
+  }
+
+  const parsedScript = cosmeticFilter.parseScript();
+
+  if (!parsedScript || !parsedScript.name) {
+    return null;
+  }
+
+  if (
+    !trustedScriptlets &&
+    (parsedScript.name === 'rpnt' ||
+      parsedScript.name === 'replace-node-text' ||
+      parsedScript.name.startsWith('trusted-'))
+  ) {
+    throw new TrustedScriptletError();
+  }
+
+  const [front] = filter.split(`#+js(${parsedScript.name}`);
+  const args = parsedScript.args.map((arg) => encodeURIComponent(arg));
+  return `${front}#+js(${[parsedScript.name, ...args].join(', ')})`;
+}
+
+function normalizeFilters(text = '', { trustedScriptlets }) {
+  const rows = text.split('\n').map((f) => f.trim());
+
+  return rows.reduce(
+    (filters, filter, index) => {
+      if (!filter) return filters;
+
+      const filterType = detectFilterType(filter, {
+        extendedNonSupportedTypes: true,
+      });
+      if (filterType === FilterType.NETWORK) {
+        filters.networkFilters.add(filter);
+      } else if (filterType === FilterType.COSMETIC) {
+        try {
+          const scriptlet = fixScriptlet(filter, trustedScriptlets);
+          filters.cosmeticFilters.add(scriptlet || filter);
+        } catch (e) {
+          if (e instanceof TrustedScriptletError) {
+            filters.errors.push(
+              `Trusted scriptlets are not allowed (${index + 1}): ${filter}`,
+            );
+          } else {
+            console.error(e);
+          }
+        }
+      } else if (
+        filterType === FilterType.NOT_SUPPORTED ||
+        filterType === FilterType.NOT_SUPPORTED_ADGUARD
+      ) {
+        filters.errors.push(`Filter not supported (${index + 1}): ${filter}`);
+      }
+      return filters;
+    },
+    {
+      networkFilters: new Set(),
+      cosmeticFilters: new Set(),
+      errors: [],
+    },
+  );
+}
+
 async function updateDNRRules(dnrRules) {
   const dynamicRules = (await chrome.declarativeNetRequest.getDynamicRules())
     .filter(({ id }) => id >= 1000000 && id < 2000000)
@@ -25,57 +123,105 @@ async function updateDNRRules(dnrRules) {
   }
 
   if (dnrRules.length) {
+    dnrRules = dnrRules.map((rule, index) => ({
+      ...rule,
+      id: 1000000 + index,
+    }));
+
     await chrome.declarativeNetRequest.updateDynamicRules({
-      addRules: dnrRules.map((rule, index) => ({
-        ...rule,
-        id: 1000000 + index,
-      })),
+      addRules: dnrRules,
     });
+
+    console.info(`Custom Filters: DNR updated with rules: ${dnrRules.length}`);
   }
+
+  return dnrRules;
 }
 
-chrome.runtime.onMessage.addListener((msg, sender, sendResponse) => {
-  if (msg.action === 'customFilters:engine') {
-    console.groupCollapsed(
-      'Custom filters: generating filters for provided input',
+function updateEngine(text) {
+  const { networkFilters, cosmeticFilters, preprocessors } = parseFilters(text);
+
+  engines.createEngine(engines.CUSTOM_ENGINE, {
+    cosmeticFilters,
+    networkFilters,
+    preprocessors,
+  });
+
+  console.info(
+    `Custom Filters: engine updated with network filters: ${networkFilters.length}, cosmetic filters: ${cosmeticFilters.length}`,
+  );
+
+  return {
+    networkFilters: networkFilters.length,
+    cosmeticFilters: cosmeticFilters.length,
+  };
+}
+async function update(text, { trustedScriptlets }) {
+  const { networkFilters, cosmeticFilters, errors } = normalizeFilters(text, {
+    trustedScriptlets,
+  });
+
+  const result = updateEngine(
+    [
+      ...(__PLATFORM__ === 'firefox' ? networkFilters : []),
+      ...cosmeticFilters,
+    ].join('\n'),
+  );
+
+  result.errors = errors;
+
+  if (__PLATFORM__ !== 'firefox') {
+    const dnrResult = await Promise.allSettled(
+      [...networkFilters].map((filter) => convert(filter)),
     );
-    console.log(msg.filters);
-    console.groupEnd();
-
-    try {
-      const {
-        cosmeticFilters,
-        networkFilters,
-        preprocessors,
-        notSupportedFilters,
-      } = parseFilters(msg.filters);
-
-      engines.createEngine(engines.CUSTOM_ENGINE, {
-        cosmeticFilters,
-        networkFilters,
-        preprocessors,
-      });
 
-      sendResponse(
-        notSupportedFilters.map(
-          (msg) => `Filter not supported: '${msg.filter}'`,
-        ),
-      );
-    } catch (e) {
-      sendResponse([e]);
+    const dnrRules = [];
+    for (const result of dnrResult) {
+      if (result.value.errors?.length) {
+        errors.push(...result.value.errors);
+      }
+
+      dnrRules.push(...result.value.rules);
     }
 
-    return false;
+    result.dnrRules = await updateDNRRules(dnrRules);
   }
 
-  if (__PLATFORM__ !== 'firefox') {
-    if (msg.action === 'customFilters:dnr') {
-      updateDNRRules(msg.dnrRules).then(() =>
-        sendResponse('DNR rules updated'),
-      );
-      return true;
+  return result;
+}
+
+observe('customFilters', async ({ enabled, trustedScriptlets }, lastValue) => {
+  // Background startup
+  if (!lastValue) {
+    const engine = await customFiltersEngine;
+    const { networkFilters, cosmeticFilters } = engine.getFilters();
+
+    // If there are filters already loaded, we can assume that
+    // filters are already added to the engine. This is specially
+    // for the case, when custom engine is reloaded because
+    // of the new adblocker version
+    if (networkFilters.length || cosmeticFilters.length) {
+      return;
     }
   }
+  // If only trustedScriptlets has changed, we don't update automatically.
+  // The user needs to click the update button.
+  else if (lastValue.enabled === enabled) {
+    return;
+  }
 
-  return false;
+  update(enabled ? (await store.resolve(CustomFilters)).text : '', {
+    trustedScriptlets,
+  });
+});
+
+chrome.runtime.onMessage.addListener((msg, sender, sendResponse) => {
+  if (msg.action === 'customFilters:update') {
+    store.resolve(Options).then((options) => {
+      // Update filters
+      update(msg.input, options.customFilters).then(sendResponse);
+    });
+
+    return true;
+  }
 });