experiment with compression

platform/chromium/vapi-cachestorage.js

@@ -1,7 +1,7 @@
 /*******************************************************************************
 
     uBlock Origin - a browser extension to block requests.
-    Copyright (C) 2016-2018 The uBlock Origin authors
+    Copyright (C) 2016-present The uBlock Origin authors
 
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -19,7 +19,7 @@
     Home: https://github.com/gorhill/uBlock
 */
 
-/* global indexedDB, IDBDatabase */
+/* global indexedDB, IDBDatabase, SnappyJS */
 
 'use strict';
 
@@ -48,13 +48,15 @@ vAPI.cacheStorage = (function() {
     }
 
     const STORAGE_NAME = 'uBlock0CacheStorage';
-    var db;
-    var pending = [];
+    let db;
+    let pending = [];
+    let textEncoder, textDecoder;
+    let api = { get, set, remove, clear, getBytesInUse, error: undefined };
 
     // prime the db so that it's ready asap for next access.
     getDb(noopfn);
 
-    return { get, set, remove, clear, getBytesInUse };
+    return api;
 
     function get(input, callback) {
         if ( typeof callback !== 'function' ) { return; }
@@ -90,8 +92,12 @@ vAPI.cacheStorage = (function() {
         callback(0);
     }
 
-    function genericErrorHandler(error) {
-        console.error('[%s]', STORAGE_NAME, error);
+    function genericErrorHandler(ev) {
+        let error = ev.target && ev.target.error;
+        if ( error && error.name === 'QuotaExceededError' ) {
+            api.error = error.name;
+        }
+        console.error('[%s]', STORAGE_NAME, error && error.name);
     }
 
     function noopfn() {
@@ -104,6 +110,71 @@ vAPI.cacheStorage = (function() {
         }
     }
 
+    function compressInput(input) {
+        let output = {};
+        for ( let key in input ) {
+            let item = input[key];
+            if ( typeof item !== 'string' || item.length < 8192 ) {
+                output[key] = item;
+                continue;
+            }
+            if ( textEncoder === undefined ) {
+                textEncoder = new TextEncoder();
+            }
+            let t0 = window.performance.now();
+            output[key] = new Blob([
+                SnappyJS.compress(textEncoder.encode(item))
+            ]);
+            let t1 = window.performance.now();
+            console.info(
+                'uBO: compressed %d bytes into %d bytes in %s ms',
+                item.length,
+                output[key].size,
+                (t1 - t0).toFixed(2)
+            );
+        }
+        return output;
+    }
+
+    function decompressInput(input, callback) {
+        let output = {};
+        let pendingItemCount = 0;
+        let countdown = function(key, ev) {
+            if ( textDecoder === undefined ) {
+                textDecoder = new TextDecoder();
+            }
+            let t0 = window.performance.now();
+            output[key] = textDecoder.decode(
+                SnappyJS.uncompress(ev.target.result)
+            );
+            let t1 = window.performance.now();
+            console.info(
+                'uBO: decompressed %d bytes into %d bytes in %s ms',
+                ev.target.result.byteLength,
+                output[key].length,
+                (t1 - t0).toFixed(2)
+            );
+            pendingItemCount -= 1;
+            if ( pendingItemCount === 0 ) {
+                callback(output);
+            }
+        };
+        for ( let key in input ) {
+            let item = input[key];
+            if ( item instanceof Blob === false ) {
+                output[key] = item;
+                continue;
+            }
+            pendingItemCount += 1;
+            let blobReader = new FileReader();
+            blobReader.onload = countdown.bind(blobReader, key);
+            blobReader.readAsArrayBuffer(item);
+        }
+        if ( pendingItemCount === 0 ) {
+            callback(output);
+        }
+    }
+
     function getDb(callback) {
         if ( pending === undefined ) {
             return callback();
@@ -174,7 +245,7 @@ vAPI.cacheStorage = (function() {
             transaction.oncomplete =
             transaction.onerror =
             transaction.onabort = function() {
-                return callback(store);
+                return decompressInput(store, callback);
             };
             var table = transaction.objectStore(STORAGE_NAME);
             for ( var key of keys ) {
@@ -197,7 +268,7 @@ vAPI.cacheStorage = (function() {
             transaction.oncomplete =
             transaction.onerror =
             transaction.onabort = function() {
-                callback(output);
+                decompressInput(output, callback);
             };
             var table = transaction.objectStore(STORAGE_NAME),
                 req = table.openCursor();
@@ -222,14 +293,14 @@ vAPI.cacheStorage = (function() {
         }
         let keys = Object.keys(input);
         if ( keys.length === 0 ) { return callback(); }
+        input = compressInput(input);
         getDb(function(db) {
             if ( !db ) { return callback(); }
-            let finish = () => {
-                if ( callback !== undefined ) {
-                    let cb = callback;
-                    callback = undefined;
-                    cb();
-                }
+            let finish = ( ) => {
+                if ( callback === undefined ) { return; }
+                let cb = callback;
+                callback = undefined;
+                cb();
             };
             try {
                 let transaction = db.transaction(STORAGE_NAME, 'readwrite');
@@ -254,17 +325,27 @@ vAPI.cacheStorage = (function() {
         if ( typeof callback !== 'function' ) {
             callback = noopfn;
         }
-        var keys = Array.isArray(input) ? input.slice() : [ input ];
+        let keys = Array.isArray(input) ? input.slice() : [ input ];
         if ( keys.length === 0 ) { return callback(); }
         getDb(function(db) {
             if ( !db ) { return callback(); }
-            var transaction = db.transaction(STORAGE_NAME, 'readwrite');
-            transaction.oncomplete =
-            transaction.onerror =
-            transaction.onabort = callback;
-            var table = transaction.objectStore(STORAGE_NAME);
-            for ( var key of keys ) {
-                table.delete(key);
+            let finish = ( ) => {
+                if ( callback === undefined ) { return; }
+                let cb = callback;
+                callback = undefined;
+                cb();
+            };
+            try {
+                let transaction = db.transaction(STORAGE_NAME, 'readwrite');
+                transaction.oncomplete =
+                transaction.onerror =
+                transaction.onabort = finish;
+                let table = transaction.objectStore(STORAGE_NAME);
+                for ( let key of keys ) {
+                    table.delete(key);
+                }
+            } catch (ex) {
+                finish();
             }
         });
     }
@@ -275,10 +356,20 @@ vAPI.cacheStorage = (function() {
         }
         getDb(function(db) {
             if ( !db ) { return callback(); }
-            var req = db.transaction(STORAGE_NAME, 'readwrite')
-                        .objectStore(STORAGE_NAME)
-                        .clear();
-            req.onsuccess = req.onerror = callback;
+            let finish = ( ) => {
+                if ( callback === undefined ) { return; }
+                let cb = callback;
+                callback = undefined;
+                cb();
+            };
+            try {
+                let req = db.transaction(STORAGE_NAME, 'readwrite')
+                            .objectStore(STORAGE_NAME)
+                            .clear();
+                req.onsuccess = req.onerror = finish;
+            } catch (ex) {
+                finish();
+            }
         });
     }
 }());

src/background.html

@@ -7,6 +7,7 @@
 <body>
 <script src="lib/punycode.js"></script>
 <script src="lib/publicsuffixlist.js"></script>
+<script src="lib/snappyjs.js"></script>
 <script src="js/vapi.js"></script>
 <script src="js/vapi-common.js"></script>
 <script src="js/vapi-background.js"></script>

src/js/background.js

@@ -138,7 +138,7 @@ var µBlock = (function() { // jshint ignore:line
         // Read-only
         systemSettings: {
             compiledMagic: 3,   // Increase when compiled format changes
-            selfieMagic: 3      // Increase when selfie format changes
+            selfieMagic: 4      // Increase when selfie format changes
         },
 
         restoreBackupSettings: {

src/js/storage.js

@@ -54,11 +54,6 @@
 µBlock.saveLocalSettings = (function() {
     let saveAfter = 4 * 60 * 1000;
 
-    let save = function(callback) {
-        this.localSettingsLastSaved = Date.now();
-        vAPI.storage.set(this.localSettings, callback);
-    };
-
     let onTimeout = ( ) => {
         let µb = µBlock;
         if ( µb.localSettingsLastModified > µb.localSettingsLastSaved ) {
@@ -69,7 +64,10 @@
 
     vAPI.setTimeout(onTimeout, saveAfter);
 
-    return save;
+    return function(callback) {
+        this.localSettingsLastSaved = Date.now();
+        vAPI.storage.set(this.localSettings, callback);
+    };
 })();
 
 /******************************************************************************/
@@ -1027,30 +1025,39 @@
 
     let create = function() {
         timer = null;
-        let selfie = {
+        let selfie = JSON.stringify({
             magic: µb.systemSettings.selfieMagic,
-            availableFilterLists: JSON.stringify(µb.availableFilterLists),
-            staticNetFilteringEngine: JSON.stringify(µb.staticNetFilteringEngine.toSelfie()),
-            redirectEngine: JSON.stringify(µb.redirectEngine.toSelfie()),
-            staticExtFilteringEngine: JSON.stringify(µb.staticExtFilteringEngine.toSelfie())
-        };
+            availableFilterLists: µb.availableFilterLists,
+            staticNetFilteringEngine: µb.staticNetFilteringEngine.toSelfie(),
+            redirectEngine: µb.redirectEngine.toSelfie(),
+            staticExtFilteringEngine: µb.staticExtFilteringEngine.toSelfie()
+        });
         vAPI.cacheStorage.set({ selfie: selfie });
     };
 
     let load = function(callback) {
         vAPI.cacheStorage.get('selfie', function(bin) {
             if (
                 bin instanceof Object === false ||
-                bin.selfie instanceof Object === false ||
-                bin.selfie.magic !== µb.systemSettings.selfieMagic ||
-                bin.selfie.redirectEngine === undefined
+                typeof bin.selfie !== 'string'
+            ) {
+                return callback(false);
+            }
+            let selfie;
+            try {
+                selfie = JSON.parse(bin.selfie);
+            } catch(ex) {
+            }
+            if (
+                selfie instanceof Object === false ||
+                selfie.magic !== µb.systemSettings.selfieMagic
             ) {
                 return callback(false);
             }
-            µb.availableFilterLists = JSON.parse(bin.selfie.availableFilterLists);
-            µb.staticNetFilteringEngine.fromSelfie(JSON.parse(bin.selfie.staticNetFilteringEngine));
-            µb.redirectEngine.fromSelfie(JSON.parse(bin.selfie.redirectEngine));
-            µb.staticExtFilteringEngine.fromSelfie(JSON.parse(bin.selfie.staticExtFilteringEngine));
+            µb.availableFilterLists = selfie.availableFilterLists;
+            µb.staticNetFilteringEngine.fromSelfie(selfie.staticNetFilteringEngine);
+            µb.redirectEngine.fromSelfie(selfie.redirectEngine);
+            µb.staticExtFilteringEngine.fromSelfie(selfie.staticExtFilteringEngine);
             callback(true);
         });
     };