Skip to content

Commit

Permalink
Refactor selfie generation into a more flexible persistence mechanism
Browse files Browse the repository at this point in the history 
The motivation is to address the higher peak memory usage at launch
time with 3rd-gen HNTrie when a selfie was present.

The selfie generation prior to this change was to collect all
filtering data into a single data structure, and then to serialize
that whole structure at once into storage (using JSON.stringify).

However, HNTrie serialization requires that a large UintArray32 be
converted into a plain JS array, which itslef would be indirectly
converted into a JSON string. This was the main reason why peak
memory usage would be higher at launch from selfie, since the JSON
string would need to be wholly unserialized into JS objects, which
themselves would need to be converted into more specialized data
structures (like that Uint32Array one).

The solution to lower peak memory usage at launch is to refactor
selfie generation to allow a more piecemeal approach: each filtering
component is given the ability to serialize itself rather than to be
forced to be embedded in the master selfie. With this approach, the
HNTrie buffer can now serialize to its own storage by converting the
buffer data directly into a string which can be directly sent to
storage. This avoiding expensive intermediate steps such as
converting into a JS array and then to a JSON string.

As part of the refactoring, there was also opportunistic code
upgrade to ES6 and Promise (eventually all of uBO's code will be
proper ES6).

Additionally, the polyfill to bring getBytesInUse() to Firefox has
been revisited to replace the rather expensive previous
implementation with an implementation with virtually no overhead.
  • Loading branch information
@gorhill
gorhill committed Feb 14, 2019
1 parent 83a3767 commit ed7e34f
Show file tree
Hide file tree
Showing 14 changed files with 585 additions and 319 deletions.
1 change: 1 addition & 0 deletions .jshintrc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
"browser": false, // global variable in Firefox, Edge
"chrome": false, // global variable in Chromium, Chrome, Opera
"Components": false, // global variable in Firefox
"log": false,
"safari": false,
"self": false,
"vAPI": false,
Expand Down
4 changes: 4 additions & 0 deletions platform/chromium/vapi-common.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,10 @@

/******************************************************************************/

vAPI.T0 = Date.now();

/******************************************************************************/

vAPI.setTimeout = vAPI.setTimeout || self.setTimeout.bind(self);

/******************************************************************************/
Expand Down
1 change: 1 addition & 0 deletions src/background.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
<title>uBlock Origin</title>
</head>
<body>
<script src="js/console.js"></script>
<script src="lib/lz4/lz4-block-codec-any.js"></script>
<script src="lib/punycode.js"></script>
<script src="lib/publicsuffixlist/publicsuffixlist.js"></script>
Expand Down
145 changes: 89 additions & 56 deletions src/js/assets.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -449,26 +449,22 @@ const assetCacheRegistryStartTime = Date.now();
let assetCacheRegistryPromise;
let assetCacheRegistry = {};

const getAssetCacheRegistry = function(callback) {
const getAssetCacheRegistry = function() {
if ( assetCacheRegistryPromise === undefined ) {
assetCacheRegistryPromise = new Promise(resolve => {
// start of executor
µBlock.cacheStorage.get('assetCacheRegistry', bin => {
if (
bin instanceof Object &&
bin.assetCacheRegistry instanceof Object
) {
assetCacheRegistry = bin.assetCacheRegistry;
}
resolve();
});
// end of executor
µBlock.cacheStorage.get('assetCacheRegistry', bin => {
if (
bin instanceof Object &&
bin.assetCacheRegistry instanceof Object
) {
assetCacheRegistry = bin.assetCacheRegistry;
}
resolve();
});
});
}

assetCacheRegistryPromise.then(( ) => {
callback(assetCacheRegistry);
});
return assetCacheRegistryPromise.then(( ) => assetCacheRegistry);
};

const saveAssetCacheRegistry = (function() {
Expand Down Expand Up @@ -513,11 +509,9 @@ const assetCacheRead = function(assetKey, callback) {
reportBack(bin[internalKey]);
};

let onReady = function() {
getAssetCacheRegistry().then(( ) => {
µBlock.cacheStorage.get(internalKey, onAssetRead);
};

getAssetCacheRegistry(onReady);
});
};

const assetCacheWrite = function(assetKey, details, callback) {
Expand All @@ -542,22 +536,35 @@ const assetCacheWrite = function(assetKey, details, callback) {
if ( details instanceof Object && typeof details.url === 'string' ) {
entry.remoteURL = details.url;
}
µBlock.cacheStorage.set({ assetCacheRegistry, [internalKey]: content });
µBlock.cacheStorage.set(
{ [internalKey]: content },
details => {
if (
details instanceof Object &&
typeof details.bytesInUse === 'number'
) {
entry.byteLength = details.bytesInUse;
}
saveAssetCacheRegistry(true);
}
);
const result = { assetKey, content };
if ( typeof callback === 'function' ) {
callback(result);
}
// https://github.com/uBlockOrigin/uBlock-issues/issues/248
fireNotification('after-asset-updated', result);
};
getAssetCacheRegistry(onReady);

getAssetCacheRegistry().then(( ) => {
µBlock.cacheStorage.get(internalKey, onReady);
});
};

const assetCacheRemove = function(pattern, callback) {
const onReady = function() {
const cacheDict = assetCacheRegistry,
removedEntries = [],
removedContent = [];
getAssetCacheRegistry().then(cacheDict => {
const removedEntries = [];
const removedContent = [];
for ( const assetKey in cacheDict ) {
if ( pattern instanceof RegExp && !pattern.test(assetKey) ) {
continue;
Expand All @@ -582,14 +589,15 @@ const assetCacheRemove = function(pattern, callback) {
{ assetKey: removedEntries[i] }
);
}
};

getAssetCacheRegistry(onReady);
});
};

const assetCacheMarkAsDirty = function(pattern, exclude, callback) {
const onReady = function() {
const cacheDict = assetCacheRegistry;
if ( typeof exclude === 'function' ) {
callback = exclude;
exclude = undefined;
}
getAssetCacheRegistry().then(cacheDict => {
let mustSave = false;
for ( const assetKey in cacheDict ) {
if ( pattern instanceof RegExp ) {
Expand Down Expand Up @@ -617,12 +625,7 @@ const assetCacheMarkAsDirty = function(pattern, exclude, callback) {
if ( typeof callback === 'function' ) {
callback();
}
};
if ( typeof exclude === 'function' ) {
callback = exclude;
exclude = undefined;
}
getAssetCacheRegistry(onReady);
});
};

/******************************************************************************/
Expand All @@ -642,12 +645,12 @@ const stringIsNotEmpty = function(s) {
**/

var readUserAsset = function(assetKey, callback) {
var reportBack = function(content) {
const readUserAsset = function(assetKey, callback) {
const reportBack = function(content) {
callback({ assetKey: assetKey, content: content });
};

var onLoaded = function(bin) {
const onLoaded = function(bin) {
if ( !bin ) { return reportBack(''); }
var content = '';
if ( typeof bin['cached_asset_content://assets/user/filters.txt'] === 'string' ) {
Expand All @@ -671,7 +674,7 @@ var readUserAsset = function(assetKey, callback) {
}
return reportBack(content);
};
var toRead = assetKey;
let toRead = assetKey;
if ( assetKey === µBlock.userFiltersPath ) {
toRead = [
assetKey,
Expand All @@ -682,7 +685,7 @@ var readUserAsset = function(assetKey, callback) {
vAPI.storage.get(toRead, onLoaded);
};

var saveUserAsset = function(assetKey, content, callback) {
const saveUserAsset = function(assetKey, content, callback) {
var bin = {};
bin[assetKey] = content;
// TODO(seamless migration):
Expand Down Expand Up @@ -711,27 +714,33 @@ api.get = function(assetKey, options, callback) {
callback = noopfunc;
}

return new Promise(resolve => {
// start of executor
if ( assetKey === µBlock.userFiltersPath ) {
readUserAsset(assetKey, callback);
readUserAsset(assetKey, details => {
callback(details);
resolve(details);
});
return;
}

var assetDetails = {},
let assetDetails = {},
contentURLs,
contentURL;

var reportBack = function(content, err) {
var details = { assetKey: assetKey, content: content };
const reportBack = function(content, err) {
const details = { assetKey: assetKey, content: content };
if ( err ) {
details.error = assetDetails.lastError = err;
} else {
assetDetails.lastError = undefined;
}
callback(details);
resolve(details);
};

var onContentNotLoaded = function() {
var isExternal;
const onContentNotLoaded = function() {
let isExternal;
while ( (contentURL = contentURLs.shift()) ) {
isExternal = reIsExternalPath.test(contentURL);
if ( isExternal === false || assetDetails.hasLocalURL !== true ) {
Expand All @@ -748,7 +757,7 @@ api.get = function(assetKey, options, callback) {
}
};

var onContentLoaded = function(details) {
const onContentLoaded = function(details) {
if ( stringIsNotEmpty(details.content) === false ) {
onContentNotLoaded();
return;
Expand All @@ -762,7 +771,7 @@ api.get = function(assetKey, options, callback) {
reportBack(details.content);
};

var onCachedContentLoaded = function(details) {
const onCachedContentLoaded = function(details) {
if ( details.content !== '' ) {
return reportBack(details.content);
}
Expand All @@ -780,11 +789,13 @@ api.get = function(assetKey, options, callback) {
};

assetCacheRead(assetKey, onCachedContentLoaded);
// end of executor
});
};

/******************************************************************************/

var getRemote = function(assetKey, callback) {
const getRemote = function(assetKey, callback) {
var assetDetails = {},
contentURLs,
contentURL;
Expand Down Expand Up @@ -852,10 +863,19 @@ var getRemote = function(assetKey, callback) {
/******************************************************************************/

api.put = function(assetKey, content, callback) {
if ( reIsUserAsset.test(assetKey) ) {
return saveUserAsset(assetKey, content, callback);
}
assetCacheWrite(assetKey, content, callback);
return new Promise(resolve => {
const onDone = function(details) {
if ( typeof callback === 'function' ) {
callback(details);
}
resolve(details);
};
if ( reIsUserAsset.test(assetKey) ) {
saveUserAsset(assetKey, content, onDone);
} else {
assetCacheWrite(assetKey, content, onDone);
}
});
};

/******************************************************************************/
Expand Down Expand Up @@ -895,14 +915,27 @@ api.metadata = function(callback) {
if ( cacheRegistryReady ) { onReady(); }
});

getAssetCacheRegistry(function() {
getAssetCacheRegistry().then(( ) => {
cacheRegistryReady = true;
if ( assetRegistryReady ) { onReady(); }
});
};

/******************************************************************************/

api.getBytesInUse = function() {
return getAssetCacheRegistry().then(cacheDict => {
let bytesUsed = 0;
for ( const assetKey in cacheDict ) {
if ( cacheDict.hasOwnProperty(assetKey) === false ) { continue; }
bytesUsed += cacheDict[assetKey].byteLength || 0;
}
return bytesUsed;
});
};

/******************************************************************************/

api.purge = assetCacheMarkAsDirty;

api.remove = function(pattern, callback) {
Expand Down Expand Up @@ -1013,7 +1046,7 @@ var updateNext = function() {
updateOne();
});

getAssetCacheRegistry(function(dict) {
getAssetCacheRegistry().then(dict => {
cacheDict = dict;
if ( !assetDict ) { return; }
updateOne();
Expand Down
14 changes: 6 additions & 8 deletions src/js/background.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,13 +46,15 @@ const µBlock = (function() { // jshint ignore:line
cacheStorageAPI: 'unset',
cacheStorageCompression: true,
cacheControlForFirefox1376932: 'no-cache, no-store, must-revalidate',
consoleLogLevel: 'unset',
debugScriptlets: false,
disableWebAssembly: false,
ignoreRedirectFilters: false,
ignoreScriptInjectFilters: false,
manualUpdateAssetFetchPeriod: 500,
popupFontSize: 'unset',
requestJournalProcessPeriod: 1000,
selfieAfter: 11,
strictBlockingBypassDuration: 120,
suspendTabsUntilReady: false,
userResourcesLocation: 'unset'
Expand Down Expand Up @@ -95,13 +97,13 @@ const µBlock = (function() { // jshint ignore:line

hiddenSettingsDefault: hiddenSettingsDefault,
hiddenSettings: (function() {
let out = Object.assign({}, hiddenSettingsDefault),
const out = Object.assign({}, hiddenSettingsDefault),
json = vAPI.localStorage.getItem('immediateHiddenSettings');
if ( typeof json === 'string' ) {
try {
let o = JSON.parse(json);
const o = JSON.parse(json);
if ( o instanceof Object ) {
for ( let k in o ) {
for ( const k in o ) {
if ( out.hasOwnProperty(k) ) {
out[k] = o[k];
}
Expand All @@ -111,8 +113,6 @@ const µBlock = (function() { // jshint ignore:line
catch(ex) {
}
}
// Remove once 1.15.12+ is widespread.
vAPI.localStorage.removeItem('hiddenSettings');
return out;
})(),

Expand All @@ -138,7 +138,7 @@ const µBlock = (function() { // jshint ignore:line
// Read-only
systemSettings: {
compiledMagic: 6, // Increase when compiled format changes
selfieMagic: 7 // Increase when selfie format changes
selfieMagic: 8 // Increase when selfie format changes
},

restoreBackupSettings: {
Expand All @@ -161,8 +161,6 @@ const µBlock = (function() { // jshint ignore:line
selectedFilterLists: [],
availableFilterLists: {},

selfieAfter: 17 * oneMinute,

pageStores: new Map(),
pageStoresToken: 0,

Expand Down
Loading

0 comments on commit ed7e34f

Please sign in to comment.