From 8ae77e6aeeaa85af335e664c2560d2afd37288c6 Mon Sep 17 00:00:00 2001 From: Raymond Hill Date: Wed, 25 Jul 2018 18:17:45 -0400 Subject: [PATCH] experiment with compression --- platform/chromium/vapi-cachestorage.js | 145 +++++-- src/background.html | 1 + src/js/background.js | 2 +- src/js/storage.js | 45 ++- src/lib/snappyjs.js | 537 +++++++++++++++++++++++++ 5 files changed, 683 insertions(+), 47 deletions(-) create mode 100644 src/lib/snappyjs.js diff --git a/platform/chromium/vapi-cachestorage.js b/platform/chromium/vapi-cachestorage.js index a25c7bc2cc394..5d6a6fbbb31ac 100644 --- a/platform/chromium/vapi-cachestorage.js +++ b/platform/chromium/vapi-cachestorage.js @@ -1,7 +1,7 @@ /******************************************************************************* uBlock Origin - a browser extension to block requests. - Copyright (C) 2016-2018 The uBlock Origin authors + Copyright (C) 2016-present The uBlock Origin authors This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -19,7 +19,7 @@ Home: https://github.com/gorhill/uBlock */ -/* global indexedDB, IDBDatabase */ +/* global indexedDB, IDBDatabase, SnappyJS */ 'use strict'; @@ -48,13 +48,15 @@ vAPI.cacheStorage = (function() { } const STORAGE_NAME = 'uBlock0CacheStorage'; - var db; - var pending = []; + let db; + let pending = []; + let textEncoder, textDecoder; + let api = { get, set, remove, clear, getBytesInUse, error: undefined }; // prime the db so that it's ready asap for next access. getDb(noopfn); - return { get, set, remove, clear, getBytesInUse }; + return api; function get(input, callback) { if ( typeof callback !== 'function' ) { return; } @@ -90,8 +92,12 @@ vAPI.cacheStorage = (function() { callback(0); } - function genericErrorHandler(error) { - console.error('[%s]', STORAGE_NAME, error); + function genericErrorHandler(ev) { + let error = ev.target && ev.target.error; + if ( error && error.name === 'QuotaExceededError' ) { + api.error = error.name; + } + console.error('[%s]', STORAGE_NAME, error && error.name); } function noopfn() { @@ -104,6 +110,71 @@ vAPI.cacheStorage = (function() { } } + function compressInput(input) { + let output = {}; + for ( let key in input ) { + let item = input[key]; + if ( typeof item !== 'string' || item.length < 8192 ) { + output[key] = item; + continue; + } + if ( textEncoder === undefined ) { + textEncoder = new TextEncoder(); + } + let t0 = window.performance.now(); + output[key] = new Blob([ + SnappyJS.compress(textEncoder.encode(item)) + ]); + let t1 = window.performance.now(); + console.info( + 'uBO: compressed %d bytes into %d bytes in %s ms', + item.length, + output[key].size, + (t1 - t0).toFixed(2) + ); + } + return output; + } + + function decompressInput(input, callback) { + let output = {}; + let pendingItemCount = 0; + let countdown = function(key, ev) { + if ( textDecoder === undefined ) { + textDecoder = new TextDecoder(); + } + let t0 = window.performance.now(); + output[key] = textDecoder.decode( + SnappyJS.uncompress(ev.target.result) + ); + let t1 = window.performance.now(); + console.info( + 'uBO: decompressed %d bytes into %d bytes in %s ms', + ev.target.result.byteLength, + output[key].length, + (t1 - t0).toFixed(2) + ); + pendingItemCount -= 1; + if ( pendingItemCount === 0 ) { + callback(output); + } + }; + for ( let key in input ) { + let item = input[key]; + if ( item instanceof Blob === false ) { + output[key] = item; + continue; + } + pendingItemCount += 1; + let blobReader = new FileReader(); + blobReader.onload = countdown.bind(blobReader, key); + blobReader.readAsArrayBuffer(item); + } + if ( pendingItemCount === 0 ) { + callback(output); + } + } + function getDb(callback) { if ( pending === undefined ) { return callback(); @@ -174,7 +245,7 @@ vAPI.cacheStorage = (function() { transaction.oncomplete = transaction.onerror = transaction.onabort = function() { - return callback(store); + return decompressInput(store, callback); }; var table = transaction.objectStore(STORAGE_NAME); for ( var key of keys ) { @@ -197,7 +268,7 @@ vAPI.cacheStorage = (function() { transaction.oncomplete = transaction.onerror = transaction.onabort = function() { - callback(output); + decompressInput(output, callback); }; var table = transaction.objectStore(STORAGE_NAME), req = table.openCursor(); @@ -222,14 +293,14 @@ vAPI.cacheStorage = (function() { } let keys = Object.keys(input); if ( keys.length === 0 ) { return callback(); } + input = compressInput(input); getDb(function(db) { if ( !db ) { return callback(); } - let finish = () => { - if ( callback !== undefined ) { - let cb = callback; - callback = undefined; - cb(); - } + let finish = ( ) => { + if ( callback === undefined ) { return; } + let cb = callback; + callback = undefined; + cb(); }; try { let transaction = db.transaction(STORAGE_NAME, 'readwrite'); @@ -254,17 +325,27 @@ vAPI.cacheStorage = (function() { if ( typeof callback !== 'function' ) { callback = noopfn; } - var keys = Array.isArray(input) ? input.slice() : [ input ]; + let keys = Array.isArray(input) ? input.slice() : [ input ]; if ( keys.length === 0 ) { return callback(); } getDb(function(db) { if ( !db ) { return callback(); } - var transaction = db.transaction(STORAGE_NAME, 'readwrite'); - transaction.oncomplete = - transaction.onerror = - transaction.onabort = callback; - var table = transaction.objectStore(STORAGE_NAME); - for ( var key of keys ) { - table.delete(key); + let finish = ( ) => { + if ( callback === undefined ) { return; } + let cb = callback; + callback = undefined; + cb(); + }; + try { + let transaction = db.transaction(STORAGE_NAME, 'readwrite'); + transaction.oncomplete = + transaction.onerror = + transaction.onabort = finish; + let table = transaction.objectStore(STORAGE_NAME); + for ( let key of keys ) { + table.delete(key); + } + } catch (ex) { + finish(); } }); } @@ -275,10 +356,20 @@ vAPI.cacheStorage = (function() { } getDb(function(db) { if ( !db ) { return callback(); } - var req = db.transaction(STORAGE_NAME, 'readwrite') - .objectStore(STORAGE_NAME) - .clear(); - req.onsuccess = req.onerror = callback; + let finish = ( ) => { + if ( callback === undefined ) { return; } + let cb = callback; + callback = undefined; + cb(); + }; + try { + let req = db.transaction(STORAGE_NAME, 'readwrite') + .objectStore(STORAGE_NAME) + .clear(); + req.onsuccess = req.onerror = finish; + } catch (ex) { + finish(); + } }); } }()); diff --git a/src/background.html b/src/background.html index cf7838befbebe..c0d97dc9d0f5b 100644 --- a/src/background.html +++ b/src/background.html @@ -7,6 +7,7 @@ + diff --git a/src/js/background.js b/src/js/background.js index d711a615a56da..e4f986b5975cf 100644 --- a/src/js/background.js +++ b/src/js/background.js @@ -138,7 +138,7 @@ var µBlock = (function() { // jshint ignore:line // Read-only systemSettings: { compiledMagic: 3, // Increase when compiled format changes - selfieMagic: 3 // Increase when selfie format changes + selfieMagic: 4 // Increase when selfie format changes }, restoreBackupSettings: { diff --git a/src/js/storage.js b/src/js/storage.js index 397cb77a0a538..d96a5dc010df9 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -54,11 +54,6 @@ µBlock.saveLocalSettings = (function() { let saveAfter = 4 * 60 * 1000; - let save = function(callback) { - this.localSettingsLastSaved = Date.now(); - vAPI.storage.set(this.localSettings, callback); - }; - let onTimeout = ( ) => { let µb = µBlock; if ( µb.localSettingsLastModified > µb.localSettingsLastSaved ) { @@ -69,7 +64,10 @@ vAPI.setTimeout(onTimeout, saveAfter); - return save; + return function(callback) { + this.localSettingsLastSaved = Date.now(); + vAPI.storage.set(this.localSettings, callback); + }; })(); /******************************************************************************/ @@ -1027,13 +1025,13 @@ let create = function() { timer = null; - let selfie = { + let selfie = JSON.stringify({ magic: µb.systemSettings.selfieMagic, - availableFilterLists: JSON.stringify(µb.availableFilterLists), - staticNetFilteringEngine: JSON.stringify(µb.staticNetFilteringEngine.toSelfie()), - redirectEngine: JSON.stringify(µb.redirectEngine.toSelfie()), - staticExtFilteringEngine: JSON.stringify(µb.staticExtFilteringEngine.toSelfie()) - }; + availableFilterLists: µb.availableFilterLists, + staticNetFilteringEngine: µb.staticNetFilteringEngine.toSelfie(), + redirectEngine: µb.redirectEngine.toSelfie(), + staticExtFilteringEngine: µb.staticExtFilteringEngine.toSelfie() + }); vAPI.cacheStorage.set({ selfie: selfie }); }; @@ -1041,16 +1039,25 @@ vAPI.cacheStorage.get('selfie', function(bin) { if ( bin instanceof Object === false || - bin.selfie instanceof Object === false || - bin.selfie.magic !== µb.systemSettings.selfieMagic || - bin.selfie.redirectEngine === undefined + typeof bin.selfie !== 'string' + ) { + return callback(false); + } + let selfie; + try { + selfie = JSON.parse(bin.selfie); + } catch(ex) { + } + if ( + selfie instanceof Object === false || + selfie.magic !== µb.systemSettings.selfieMagic ) { return callback(false); } - µb.availableFilterLists = JSON.parse(bin.selfie.availableFilterLists); - µb.staticNetFilteringEngine.fromSelfie(JSON.parse(bin.selfie.staticNetFilteringEngine)); - µb.redirectEngine.fromSelfie(JSON.parse(bin.selfie.redirectEngine)); - µb.staticExtFilteringEngine.fromSelfie(JSON.parse(bin.selfie.staticExtFilteringEngine)); + µb.availableFilterLists = selfie.availableFilterLists; + µb.staticNetFilteringEngine.fromSelfie(selfie.staticNetFilteringEngine); + µb.redirectEngine.fromSelfie(selfie.redirectEngine); + µb.staticExtFilteringEngine.fromSelfie(selfie.staticExtFilteringEngine); callback(true); }); }; diff --git a/src/lib/snappyjs.js b/src/lib/snappyjs.js new file mode 100644 index 0000000000000..357edd69c9562 --- /dev/null +++ b/src/lib/snappyjs.js @@ -0,0 +1,537 @@ +/** + * Modules in this bundle + * @license + * + * snappyjs: + * license: MIT (http://opensource.org/licenses/MIT) + * author: Zhipeng Jia + * version: 0.6.0 + * + * This header is generated by licensify (https://github.com/twada/licensify) + */ +(function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require=="function"&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);var f=new Error("Cannot find module '"+o+"'");throw f.code="MODULE_NOT_FOUND",f}var l=n[o]={exports:{}};t[o][0].call(l.exports,function(e){var n=t[o][1][e];return s(n?n:e)},l,l.exports,e,t,n,r)}return n[o].exports}var i=typeof require=="function"&&require;for(var o=0;o>> hashFuncShift +} + +function load32 (array, pos) { + return array[pos] + (array[pos + 1] << 8) + (array[pos + 2] << 16) + (array[pos + 3] << 24) +} + +function equals32 (array, pos1, pos2) { + return array[pos1] === array[pos2] && + array[pos1 + 1] === array[pos2 + 1] && + array[pos1 + 2] === array[pos2 + 2] && + array[pos1 + 3] === array[pos2 + 3] +} + +function copyBytes (fromArray, fromPos, toArray, toPos, length) { + var i + for (i = 0; i < length; i++) { + toArray[toPos + i] = fromArray[fromPos + i] + } +} + +function emitLiteral (input, ip, len, output, op) { + if (len <= 60) { + output[op] = (len - 1) << 2 + op += 1 + } else if (len < 256) { + output[op] = 60 << 2 + output[op + 1] = len - 1 + op += 2 + } else { + output[op] = 61 << 2 + output[op + 1] = (len - 1) & 0xff + output[op + 2] = (len - 1) >>> 8 + op += 3 + } + copyBytes(input, ip, output, op, len) + return op + len +} + +function emitCopyLessThan64 (output, op, offset, len) { + if (len < 12 && offset < 2048) { + output[op] = 1 + ((len - 4) << 2) + ((offset >>> 8) << 5) + output[op + 1] = offset & 0xff + return op + 2 + } else { + output[op] = 2 + ((len - 1) << 2) + output[op + 1] = offset & 0xff + output[op + 2] = offset >>> 8 + return op + 3 + } +} + +function emitCopy (output, op, offset, len) { + while (len >= 68) { + op = emitCopyLessThan64(output, op, offset, 64) + len -= 64 + } + if (len > 64) { + op = emitCopyLessThan64(output, op, offset, 60) + len -= 60 + } + return emitCopyLessThan64(output, op, offset, len) +} + +function compressFragment (input, ip, inputSize, output, op) { + var hashTableBits = 1 + while ((1 << hashTableBits) <= inputSize && + hashTableBits <= MAX_HASH_TABLE_BITS) { + hashTableBits += 1 + } + hashTableBits -= 1 + var hashFuncShift = 32 - hashTableBits + + if (typeof globalHashTables[hashTableBits] === 'undefined') { + globalHashTables[hashTableBits] = new Uint16Array(1 << hashTableBits) + } + var hashTable = globalHashTables[hashTableBits] + var i + for (i = 0; i < hashTable.length; i++) { + hashTable[i] = 0 + } + + var ipEnd = ip + inputSize + var ipLimit + var baseIp = ip + var nextEmit = ip + + var hash, nextHash + var nextIp, candidate, skip + var bytesBetweenHashLookups + var base, matched, offset + var prevHash, curHash + var flag = true + + var INPUT_MARGIN = 15 + if (inputSize >= INPUT_MARGIN) { + ipLimit = ipEnd - INPUT_MARGIN + + ip += 1 + nextHash = hashFunc(load32(input, ip), hashFuncShift) + + while (flag) { + skip = 32 + nextIp = ip + do { + ip = nextIp + hash = nextHash + bytesBetweenHashLookups = skip >>> 5 + skip += 1 + nextIp = ip + bytesBetweenHashLookups + if (ip > ipLimit) { + flag = false + break + } + nextHash = hashFunc(load32(input, nextIp), hashFuncShift) + candidate = baseIp + hashTable[hash] + hashTable[hash] = ip - baseIp + } while (!equals32(input, ip, candidate)) + + if (!flag) { + break + } + + op = emitLiteral(input, nextEmit, ip - nextEmit, output, op) + + do { + base = ip + matched = 4 + while (ip + matched < ipEnd && input[ip + matched] === input[candidate + matched]) { + matched += 1 + } + ip += matched + offset = base - candidate + op = emitCopy(output, op, offset, matched) + + nextEmit = ip + if (ip >= ipLimit) { + flag = false + break + } + prevHash = hashFunc(load32(input, ip - 1), hashFuncShift) + hashTable[prevHash] = ip - 1 - baseIp + curHash = hashFunc(load32(input, ip), hashFuncShift) + candidate = baseIp + hashTable[curHash] + hashTable[curHash] = ip - baseIp + } while (equals32(input, ip, candidate)) + + if (!flag) { + break + } + + ip += 1 + nextHash = hashFunc(load32(input, ip), hashFuncShift) + } + } + + if (nextEmit < ipEnd) { + op = emitLiteral(input, nextEmit, ipEnd - nextEmit, output, op) + } + + return op +} + +function putVarint (value, output, op) { + do { + output[op] = value & 0x7f + value = value >>> 7 + if (value > 0) { + output[op] += 0x80 + } + op += 1 + } while (value > 0) + return op +} + +function SnappyCompressor (uncompressed) { + this.array = uncompressed +} + +SnappyCompressor.prototype.maxCompressedLength = function () { + var sourceLen = this.array.length + return 32 + sourceLen + Math.floor(sourceLen / 6) +} + +SnappyCompressor.prototype.compressToBuffer = function (outBuffer) { + var array = this.array + var length = array.length + var pos = 0 + var outPos = 0 + + var fragmentSize + + outPos = putVarint(length, outBuffer, outPos) + while (pos < length) { + fragmentSize = Math.min(length - pos, BLOCK_SIZE) + outPos = compressFragment(array, pos, fragmentSize, outBuffer, outPos) + pos += fragmentSize + } + + return outPos +} + +exports.SnappyCompressor = SnappyCompressor + +},{}],4:[function(require,module,exports){ +// The MIT License (MIT) +// +// Copyright (c) 2016 Zhipeng Jia +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +'use strict' + +var WORD_MASK = [0, 0xff, 0xffff, 0xffffff, 0xffffffff] + +function copyBytes (fromArray, fromPos, toArray, toPos, length) { + var i + for (i = 0; i < length; i++) { + toArray[toPos + i] = fromArray[fromPos + i] + } +} + +function selfCopyBytes (array, pos, offset, length) { + var i + for (i = 0; i < length; i++) { + array[pos + i] = array[pos - offset + i] + } +} + +function SnappyDecompressor (compressed) { + this.array = compressed + this.pos = 0 +} + +SnappyDecompressor.prototype.readUncompressedLength = function () { + var result = 0 + var shift = 0 + var c, val + while (shift < 32 && this.pos < this.array.length) { + c = this.array[this.pos] + this.pos += 1 + val = c & 0x7f + if (((val << shift) >>> shift) !== val) { + return -1 + } + result |= val << shift + if (c < 128) { + return result + } + shift += 7 + } + return -1 +} + +SnappyDecompressor.prototype.uncompressToBuffer = function (outBuffer) { + var array = this.array + var arrayLength = array.length + var pos = this.pos + var outPos = 0 + + var c, len, smallLen + var offset + + while (pos < array.length) { + c = array[pos] + pos += 1 + if ((c & 0x3) === 0) { + // Literal + len = (c >>> 2) + 1 + if (len > 60) { + if (pos + 3 >= arrayLength) { + return false + } + smallLen = len - 60 + len = array[pos] + (array[pos + 1] << 8) + (array[pos + 2] << 16) + (array[pos + 3] << 24) + len = (len & WORD_MASK[smallLen]) + 1 + pos += smallLen + } + if (pos + len > arrayLength) { + return false + } + copyBytes(array, pos, outBuffer, outPos, len) + pos += len + outPos += len + } else { + switch (c & 0x3) { + case 1: + len = ((c >>> 2) & 0x7) + 4 + offset = array[pos] + ((c >>> 5) << 8) + pos += 1 + break + case 2: + if (pos + 1 >= arrayLength) { + return false + } + len = (c >>> 2) + 1 + offset = array[pos] + (array[pos + 1] << 8) + pos += 2 + break + case 3: + if (pos + 3 >= arrayLength) { + return false + } + len = (c >>> 2) + 1 + offset = array[pos] + (array[pos + 1] << 8) + (array[pos + 2] << 16) + (array[pos + 3] << 24) + pos += 4 + break + default: + break + } + if (offset === 0 || offset > outPos) { + return false + } + selfCopyBytes(outBuffer, outPos, offset, len) + outPos += len + } + } + return true +} + +exports.SnappyDecompressor = SnappyDecompressor + +},{}]},{},[1]);