From f0db3f46f2a9bf368cdb7ac01d508f827d129a62 Mon Sep 17 00:00:00 2001 From: Nicolas Humbert Date: Wed, 26 Jul 2023 17:28:01 -0400 Subject: [PATCH] ARSN-355 List lifecycle non-current versions supports V0 --- lib/algos/list/delimiterNonCurrent.js | 135 +++++++------------------- 1 file changed, 34 insertions(+), 101 deletions(-) diff --git a/lib/algos/list/delimiterNonCurrent.js b/lib/algos/list/delimiterNonCurrent.js index 334f6e79c..79fd61ed5 100644 --- a/lib/algos/list/delimiterNonCurrent.js +++ b/lib/algos/list/delimiterNonCurrent.js @@ -1,10 +1,9 @@ -'use strict'; // eslint-disable-line strict -const Delimiter = require('./delimiter').Delimiter; -const VSConst = require('../../versioning/constants').VersioningConstants; -const { inc, FILTER_ACCEPT, FILTER_END, SKIP_NONE } = require('./tools'); -const VID_SEP = VSConst.VersionId.Separator; -const Version = require('../../versioning/Version').Version; -const { DbPrefixes } = VSConst; +const DelimiterVersions = require('./delimiterVersions').DelimiterVersions; +// const VSConst = require('../../versioning/constants').VersioningConstants; +const { FILTER_ACCEPT, FILTER_END, FILTER_SKIP } = require('./tools'); +// const VID_SEP = VSConst.VersionId.Separator; +// const Version = require('../../versioning/Version').Version; +// const { DbPrefixes } = VSConst; // TODO: find an acceptable timeout value. const DELIMITER_TIMEOUT_MS = 10 * 1000; // 10s @@ -14,7 +13,7 @@ const TRIM_METADATA_MIN_BLOB_SIZE = 10000; * Handle object listing with parameters. This extends the base class Delimiter * to return the raw non-current versions objects. */ -class DelimiterNonCurrent extends Delimiter { +class DelimiterNonCurrent extends DelimiterVersions { /** * Delimiter listing of non-current versions. * @param {Object} parameters - listing parameters @@ -39,67 +38,11 @@ class DelimiterNonCurrent extends Delimiter { // internal state this.staleDate = null; - this.masterKey = undefined; - this.masterVersionId = undefined; // used for monitoring this.evaluatedKeys = 0; } - skippingV1() { - return SKIP_NONE; - } - - compareObjects(masterObj, versionObj) { - const masterKey = masterObj.key.slice(DbPrefixes.Master.length); - const versionKey = versionObj.key.slice(DbPrefixes.Version.length); - return masterKey < versionKey ? -1 : 1; - } - - genMDParamsV1() { - const vParams = { - gte: DbPrefixes.Version, - lt: inc(DbPrefixes.Version), - }; - - const mParams = { - gte: DbPrefixes.Master, - lt: inc(DbPrefixes.Master), - }; - - if (this.prefix) { - const masterWithPrefix = `${DbPrefixes.Master}${this.prefix}`; - mParams.gte = masterWithPrefix; - mParams.lt = inc(masterWithPrefix); - - const versionWithPrefix = `${DbPrefixes.Version}${this.prefix}`; - vParams.gte = versionWithPrefix; - vParams.lt = inc(versionWithPrefix); - } - - if (this.keyMarker && `${DbPrefixes.Version}${this.keyMarker}` >= vParams.gte) { - if (this.versionIdMarker) { - const keyMarkerWithVersionId = `${this.keyMarker}${VID_SEP}${this.versionIdMarker}`; - // versionIdMarker should always come with keyMarker but may not be the other way around. - // NOTE: "gte" (instead of "gt") is used to include the last version of the "previous" - // truncated listing when a versionId marker is specified. - // This "previous"/"already evaluated" version will be used to retrieve the stale date and - // skipped to not evaluate the same key twice in the addContents() method. - vParams.gte = `${DbPrefixes.Version}${keyMarkerWithVersionId}`; - mParams.gte = `${DbPrefixes.Master}${keyMarkerWithVersionId}`; - } else { - delete vParams.gte; - delete mParams.gte; - vParams.gt = DbPrefixes.Version + inc(this.keyMarker + VID_SEP); - mParams.gt = DbPrefixes.Master + inc(this.keyMarker + VID_SEP); - } - } - - this.start = Date.now(); - - return [mParams, vParams]; - } - getLastModified(value) { let lastModified; try { @@ -115,31 +58,25 @@ class DelimiterNonCurrent extends Delimiter { return lastModified; } - parseKey(fullKey) { - const versionIdIndex = fullKey.indexOf(VID_SEP); - if (versionIdIndex === -1) { - return { key: fullKey }; - } - const nonversionedKey = fullKey.slice(0, versionIdIndex); - const versionId = fullKey.slice(versionIdIndex + 1); - return { key: nonversionedKey, versionId }; - } - - /** - * Filter to apply on each iteration - * @param {Object} obj - The key and value of the element - * @param {String} obj.key - The key of the element - * @param {String} obj.value - The value of the element - * @return {number} - indicates if iteration should continue - */ - filter(obj) { - const value = obj.value; - // NOTE: this check on PHD is only useful for Artesca, S3C - // does not use PHDs in V1 format - if (Version.isPHD(value)) { - return FILTER_ACCEPT; + keyHandler_SkippingVersions(key, value) { + const { key: nonversionedKey, versionId } = this.parseKey(key); + if (nonversionedKey === this.keyMarker) { + // since the nonversioned key equals the marker, there is + // necessarily a versionId in this key + const _versionId = versionId; + if (_versionId < this.versionIdMarker) { + // skip all versions until marker + return FILTER_SKIP; + } + // if (_versionId === this.versionIdMarker) { + // // nothing left to skip, so return ACCEPT, but don't add this version + // return FILTER_ACCEPT; + // } } - return super.filter(obj); + this.setState({ + id: 1 /* NotSkipping */, + }); + return this.handleKey(key, value); } /** @@ -158,11 +95,12 @@ class DelimiterNonCurrent extends Delimiter { * - no more metadata key is left to be processed * - the listing reaches the maximum number of key to be returned * - the internal timeout is reached - * @param {String} keyVersionSuffix - The key to add + * @param {String} key - The key to add + * @param {String} versionId - The version id * @param {String} value - The value of the key * @return {number} - indicates if iteration should continue */ - addContents(keyVersionSuffix, value) { + addContents(key, versionId, value) { if (this._reachedMaxKeys()) { return FILTER_END; } @@ -178,28 +116,23 @@ class DelimiterNonCurrent extends Delimiter { } ++this.evaluatedKeys; - const { key, versionId } = this.parseKey(keyVersionSuffix); - this.NextKeyMarker = key; this.NextVersionIdMarker = versionId; // The master key serves two purposes: // - It retrieves the expiration date for the previous version that is no longer current. // - It excludes the current version from the list. - const isMasterKey = versionId === undefined; + const isMasterKey = this.masterKey === key && this.masterVersionId === versionId; if (isMasterKey) { - this.masterKey = key; - this.masterVersionId = Version.from(value).getVersionId() || 'null'; - this.staleDate = this.getLastModified(value); return FILTER_ACCEPT; } - const isCurrentVersion = this.masterKey === key && this.masterVersionId === versionId; - if (isCurrentVersion) { - // filter out the master version - return FILTER_ACCEPT; - } + // const isCurrentVersion = this.masterKey === key && this.masterVersionId === versionId; + // if (isCurrentVersion) { + // // filter out the master version + // return FILTER_ACCEPT; + // } // The following version is pushed only: // - if the "stale date" (picked up from the previous version) is available (JSON.parse has not failed),