Skip to content

Commit

Permalink
Fine-tune details about when differential update should kick in
Browse files Browse the repository at this point in the history 
Manual update of one or more lists will cause the most recent version
of these lists to be fetched from the "origin" server, and since the
lists from "origin" servers cannot be updated through differential
update, the lists will be subsequently updated according to their
`Expires` directive.

When the lists are auto-updated, the "CDN" servers will be used,
and as a result the lists will start to be updated trhough
differential updates every 6-hour (currently).

Thus it is recommended and optimal to let the lists auto-update,
since you will benefit from a much shorter delay to get up-to-date
lists (i.e. every 6-hour instead of every 6-day).

You can force the auto-updater to fetch all the lists by clicking
"Purge all caches", then restart uBO without clicking "Update".
This will cause uBO to perform an emergency auto-update at restart
time, after which you will have all the lists which are candidates
for differential update.

The "Update now" button in the "Support" pane will also cause lists
to be fetched from their "origin" server.
  • Loading branch information
@gorhill
gorhill committed Nov 1, 2023
1 parent 2acf8a6 commit 69fce3a
Show file tree
Hide file tree
Showing 3 changed files with 66 additions and 66 deletions.
122 changes: 59 additions & 63 deletions src/js/assets.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,10 @@
import cacheStorage from './cachestorage.js';
import logger from './logger.js';
import µb from './background.js';
import { ubolog } from './console.js';
import { i18n$ } from './i18n.js';
import * as sfp from './static-filtering-parser.js';
import { ubolog } from './console.js';
import { orphanizeString, } from './text-utils.js';

/******************************************************************************/

Expand All @@ -47,6 +48,8 @@ let remoteServerFriendly = false;

/******************************************************************************/

const stringIsNotEmpty = s => typeof s === 'string' && s !== '';

const parseExpires = s => {
const matches = s.match(/(\d+)\s*([dh])?/i);
if ( matches === null ) { return 0; }
Expand All @@ -71,7 +74,7 @@ const extractMetadataFromList = (content, fields) => {
field = field.toLowerCase().replace(
/-[a-z]/g, s => s.charAt(1).toUpperCase()
);
out[field] = value;
out[field] = value && orphanizeString(value);
}
// Pre-process known fields
if ( out.lastModified ) {
Expand Down Expand Up @@ -169,7 +172,44 @@ const isDiffUpdatableAsset = content => {
/^[^%].*[^%]$/.test(data.diffPath);
};

const stringIsNotEmpty = s => typeof s === 'string' && s !== '';
/******************************************************************************/

// favorLocal: avoid making network requests whenever possible
// favorOrigin: avoid using CDN URLs whenever possible

const getContentURLs = (assetKey, options = {}) => {
const contentURLs = [];
const entry = assetSourceRegistry[assetKey];
if ( entry instanceof Object === false ) { return contentURLs; }
if ( typeof entry.contentURL === 'string' ) {
contentURLs.push(entry.contentURL);
} else if ( Array.isArray(entry.contentURL) ) {
contentURLs.push(...entry.contentURL);
} else if ( reIsExternalPath.test(assetKey) ) {
contentURLs.push(assetKey);
}
if ( options.favorLocal ) {
contentURLs.sort((a, b) => {
if ( reIsExternalPath.test(a) ) { return 1; }
if ( reIsExternalPath.test(b) ) { return -1; }
return 0;
});
}
if ( Array.isArray(entry.cdnURLs) ) {
const cdnURLs = entry.cdnURLs.slice();
for ( let i = 0, n = cdnURLs.length; i < n; i++ ) {
const j = Math.floor(Math.random() * n);
if ( j === i ) { continue; }
[ cdnURLs[j], cdnURLs[i] ] = [ cdnURLs[i], cdnURLs[j] ];
}
if ( options.favorLocal || options.favorOrigin ) {
contentURLs.push(...cdnURLs);
} else {
contentURLs.unshift(...cdnURLs);
}
}
return contentURLs;
};

/******************************************************************************/

Expand Down Expand Up @@ -917,28 +957,17 @@ assets.get = async function(assetKey, options = {}) {
}

const assetRegistry = await getAssetSourceRegistry();

assetDetails = assetRegistry[assetKey] || {};
const contentURLs = [];
if ( typeof assetDetails.contentURL === 'string' ) {
contentURLs.push(assetDetails.contentURL);
} else if ( Array.isArray(assetDetails.contentURL) ) {
contentURLs.push(...assetDetails.contentURL);
} else if ( reIsExternalPath.test(assetKey) ) {

const contentURLs = getContentURLs(assetKey, options);
if ( contentURLs.length === 0 && reIsExternalPath.test(assetKey) ) {
assetDetails.content = 'filters';
contentURLs.push(assetKey);
}

// https://github.com/uBlockOrigin/uBlock-issues/issues/1566#issuecomment-826473517
// Use CDN URLs as fall back URLs.
if ( Array.isArray(assetDetails.cdnURLs) ) {
contentURLs.push(...assetDetails.cdnURLs);
}

let error = 'ENOTFOUND';
for ( const contentURL of contentURLs ) {
if ( reIsExternalPath.test(contentURL) && assetDetails.hasLocalURL ) {
continue;
}
const details = assetDetails.content === 'filters'
? await assets.fetchFilterList(contentURL)
: await assets.fetchText(contentURL);
Expand Down Expand Up @@ -966,7 +995,7 @@ assets.get = async function(assetKey, options = {}) {

/******************************************************************************/

async function getRemote(assetKey) {
async function getRemote(assetKey, options = {}) {
const [
assetDetails = {},
cacheDetails = {},
Expand All @@ -978,56 +1007,19 @@ async function getRemote(assetKey) {
let error;
let stale = false;

const reportBack = function(content, err) {
const details = { assetKey, content };
if ( err ) {
const reportBack = function(content, url = '', err = '') {
const details = { assetKey, content, url };
if ( err !== '') {
details.error = assetDetails.lastError = err;
} else {
assetDetails.lastError = undefined;
}
return details;
};

const contentURLs = [];
if ( typeof assetDetails.contentURL === 'string' ) {
contentURLs.push(assetDetails.contentURL);
} else if ( Array.isArray(assetDetails.contentURL) ) {
contentURLs.push(...assetDetails.contentURL);
}

// If asked to be gentle on remote servers, favour using dedicated CDN
// servers. If more than one CDN server is present, randomly shuffle the
// set of servers so as to spread the bandwidth burden.
//
// https://github.com/uBlockOrigin/uBlock-issues/issues/1566#issuecomment-826473517
// In case of manual update, use CDNs URLs as fall back URLs.
if ( Array.isArray(assetDetails.cdnURLs) ) {
const cdnURLs = assetDetails.cdnURLs.slice();
for ( let i = 0, n = cdnURLs.length; i < n; i++ ) {
const j = Math.floor(Math.random() * n);
if ( j === i ) { continue; }
[ cdnURLs[j], cdnURLs[i] ] = [ cdnURLs[i], cdnURLs[j] ];
}
if ( remoteServerFriendly ) {
contentURLs.unshift(...cdnURLs);
} else {
contentURLs.push(...cdnURLs);
}
}

for ( let contentURL of contentURLs ) {
for ( const contentURL of getContentURLs(assetKey, options) ) {
if ( reIsExternalPath.test(contentURL) === false ) { continue; }

// This will force uBO to fetch the proper version according to whether
// the dev build is being used. This can be removed when execution of
// this code path is widespread for dev build revisions of uBO.
if ( assetKey === 'assets.json' ) {
contentURL = contentURL.replace(
/\/assets\/assets\.json$/,
µb.assetsJsonPath
);
}

const result = assetDetails.content === 'filters'
? await assets.fetchFilterList(contentURL)
: await assets.fetchText(contentURL);
Expand Down Expand Up @@ -1066,12 +1058,12 @@ async function getRemote(assetKey) {
}

registerAssetSource(assetKey, { birthtime: undefined, error: undefined });
return reportBack(result.content);
return reportBack(result.content, contentURL);
}

if ( error !== undefined ) {
registerAssetSource(assetKey, { error: { time: Date.now(), error } });
return reportBack('', 'ENOTFOUND');
return reportBack('', '', 'ENOTFOUND');
}

if ( stale ) {
Expand Down Expand Up @@ -1194,6 +1186,8 @@ const getAssetDiffDetails = assetKey => {
};

async function diffUpdater() {
if ( updaterAuto === false ) { return; }
if ( µb.hiddenSettings.differentialUpdate === false ) { return; }
const toUpdate = await getUpdateCandidates();
const now = Date.now();
const toHardUpdate = [];
Expand Down Expand Up @@ -1298,6 +1292,7 @@ async function diffUpdater() {

function updateFirst() {
ubolog('Updater: cycle start');
ubolog('Updater: Fetch from ', updaterAuto ? 'CDNs' : 'origin');
updaterStatus = 'updating';
updaterFetched.clear();
updaterUpdated.length = 0;
Expand Down Expand Up @@ -1367,7 +1362,7 @@ async function updateNext() {

let result;
if ( assetKey !== 'assets.json' || µb.hiddenSettings.debugAssetsJson !== true ) {
result = await getRemote(assetKey);
result = await getRemote(assetKey, { favorOrigin: updaterAuto === false });
} else {
result = await assets.fetchText(µb.assetsJsonPath);
result.assetKey = 'assets.json';
Expand Down Expand Up @@ -1396,6 +1391,7 @@ function updateDone() {
updaterFetched.clear();
updaterUpdated.length = 0;
updaterStatus = undefined;
updaterAuto = false;
updaterAssetDelay = updaterAssetDelayDefault;
ubolog('Updater: cycle end');
if ( assetKeys.length ) {
Expand Down
5 changes: 3 additions & 2 deletions src/js/background.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,9 @@ const hiddenSettingsDefault = {
allowGenericProceduralFilters: false,
assetFetchTimeout: 30,
autoCommentFilterTemplate: '{{date}} {{origin}}',
autoUpdateAssetFetchPeriod: 60,
autoUpdateAssetFetchPeriod: 15,
autoUpdateDelayAfterLaunch: 105,
autoUpdatePeriod: 2,
autoUpdatePeriod: 1,
benchmarkDatasetURL: 'unset',
blockingProfiles: '11111/#F00 11010/#C0F 11001/#00F 00001',
cacheStorageAPI: 'unset',
Expand All @@ -69,6 +69,7 @@ const hiddenSettingsDefault = {
debugAssetsJson: false,
debugScriptlets: false,
debugScriptletInjector: false,
differentialUpdate: true,
disableWebAssembly: false,
extensionUpdateForceReload: false,
filterAuthorMode: false,
Expand Down
5 changes: 4 additions & 1 deletion src/js/storage.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -981,7 +981,10 @@ import {
return { assetKey, content: '' };
}

const rawDetails = await io.get(assetKey, { silent: true });
const rawDetails = await io.get(assetKey, {
favorLocal: this.readyToFilter !== true,
silent: true,
});
// Compiling an empty string results in an empty string.
if ( rawDetails.content === '' ) {
rawDetails.assetKey = assetKey;
Expand Down

0 comments on commit 69fce3a

Please sign in to comment.