Skip to content

Commit

Permalink
ARSN-355 List lifecycle non-current versions supports V0
Browse files Browse the repository at this point in the history
  • Loading branch information
@nicolas2bert
nicolas2bert committed Jul 27, 2023
1 parent 7becb8e commit a5032ee
Showing 1 changed file with 43 additions and 119 deletions.
162 changes: 43 additions & 119 deletions lib/algos/list/delimiterNonCurrent.js
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
'use strict'; // eslint-disable-line strict
const Delimiter = require('./delimiter').Delimiter;
const VSConst = require('../../versioning/constants').VersioningConstants;
const { inc, FILTER_ACCEPT, FILTER_END, SKIP_NONE } = require('./tools');
const VID_SEP = VSConst.VersionId.Separator;
const Version = require('../../versioning/Version').Version;
const { DbPrefixes } = VSConst;
const DelimiterVersions = require('./delimiterVersions').DelimiterVersions;
// const VSConst = require('../../versioning/constants').VersioningConstants;
const { FILTER_ACCEPT, FILTER_END, FILTER_SKIP } = require('./tools');
// const VID_SEP = VSConst.VersionId.Separator;
// const Version = require('../../versioning/Version').Version;
// const { DbPrefixes } = VSConst;

// TODO: find an acceptable timeout value.
const DELIMITER_TIMEOUT_MS = 10 * 1000; // 10s
Expand All @@ -14,7 +13,7 @@ const TRIM_METADATA_MIN_BLOB_SIZE = 10000;
* Handle object listing with parameters. This extends the base class Delimiter
* to return the raw non-current versions objects.
*/
class DelimiterNonCurrent extends Delimiter {
class DelimiterNonCurrent extends DelimiterVersions {
/**
* Delimiter listing of non-current versions.
* @param {Object} parameters - listing parameters
Expand All @@ -28,78 +27,16 @@ class DelimiterNonCurrent extends Delimiter {
constructor(parameters, logger, vFormat) {
super(parameters, logger, vFormat);

this.versionIdMarker = parameters.versionIdMarker;
this.beforeDate = parameters.beforeDate;
this.keyMarker = parameters.keyMarker;
this.excludedDataStoreName = parameters.excludedDataStoreName;
this.NextKeyMarker = null;

this.skipping = this.skippingV1;
this.genMDParams = this.genMDParamsV1;

// internal state
this.staleDate = null;
this.masterKey = undefined;
this.masterVersionId = undefined;

// used for monitoring
this.evaluatedKeys = 0;
}

skippingV1() {
return SKIP_NONE;
}

compareObjects(masterObj, versionObj) {
const masterKey = masterObj.key.slice(DbPrefixes.Master.length);
const versionKey = versionObj.key.slice(DbPrefixes.Version.length);
return masterKey < versionKey ? -1 : 1;
}

genMDParamsV1() {
const vParams = {
gte: DbPrefixes.Version,
lt: inc(DbPrefixes.Version),
};

const mParams = {
gte: DbPrefixes.Master,
lt: inc(DbPrefixes.Master),
};

if (this.prefix) {
const masterWithPrefix = `${DbPrefixes.Master}${this.prefix}`;
mParams.gte = masterWithPrefix;
mParams.lt = inc(masterWithPrefix);

const versionWithPrefix = `${DbPrefixes.Version}${this.prefix}`;
vParams.gte = versionWithPrefix;
vParams.lt = inc(versionWithPrefix);
}

if (this.keyMarker && `${DbPrefixes.Version}${this.keyMarker}` >= vParams.gte) {
if (this.versionIdMarker) {
const keyMarkerWithVersionId = `${this.keyMarker}${VID_SEP}${this.versionIdMarker}`;
// versionIdMarker should always come with keyMarker but may not be the other way around.
// NOTE: "gte" (instead of "gt") is used to include the last version of the "previous"
// truncated listing when a versionId marker is specified.
// This "previous"/"already evaluated" version will be used to retrieve the stale date and
// skipped to not evaluate the same key twice in the addContents() method.
vParams.gte = `${DbPrefixes.Version}${keyMarkerWithVersionId}`;
mParams.gte = `${DbPrefixes.Master}${keyMarkerWithVersionId}`;
} else {
delete vParams.gte;
delete mParams.gte;
vParams.gt = DbPrefixes.Version + inc(this.keyMarker + VID_SEP);
mParams.gt = DbPrefixes.Master + inc(this.keyMarker + VID_SEP);
}
}

this.start = Date.now();

return [mParams, vParams];
}

getLastModified(value) {
let lastModified;
try {
Expand All @@ -115,31 +52,25 @@ class DelimiterNonCurrent extends Delimiter {
return lastModified;
}

parseKey(fullKey) {
const versionIdIndex = fullKey.indexOf(VID_SEP);
if (versionIdIndex === -1) {
return { key: fullKey };
}
const nonversionedKey = fullKey.slice(0, versionIdIndex);
const versionId = fullKey.slice(versionIdIndex + 1);
return { key: nonversionedKey, versionId };
}

/**
* Filter to apply on each iteration
* @param {Object} obj - The key and value of the element
* @param {String} obj.key - The key of the element
* @param {String} obj.value - The value of the element
* @return {number} - indicates if iteration should continue
*/
filter(obj) {
const value = obj.value;
// NOTE: this check on PHD is only useful for Artesca, S3C
// does not use PHDs in V1 format
if (Version.isPHD(value)) {
return FILTER_ACCEPT;
keyHandler_SkippingVersions(key, value) {

Check failure on line 55 in lib/algos/list/delimiterNonCurrent.js

View workflow job for this annotation

GitHub Actions / test 

Identifier 'keyHandler_SkippingVersions' is not in camel case
const { key: nonversionedKey, versionId } = this.parseKey(key);
if (nonversionedKey === this.keyMarker) {
// since the nonversioned key equals the marker, there is
// necessarily a versionId in this key
const _versionId = versionId;
if (_versionId < this.versionIdMarker) {
// skip all versions until marker
return FILTER_SKIP;
}
// if (_versionId === this.versionIdMarker) {
// // nothing left to skip, so return ACCEPT, but don't add this version
// return FILTER_ACCEPT;
// }
}
return super.filter(obj);
this.setState({
id: 1 /* NotSkipping */,
});
return this.handleKey(key, value);
}

/**
Expand All @@ -158,11 +89,12 @@ class DelimiterNonCurrent extends Delimiter {
* - no more metadata key is left to be processed
* - the listing reaches the maximum number of key to be returned
* - the internal timeout is reached
* @param {String} keyVersionSuffix - The key to add
* @param {String} key - The key to add
* @param {String} versionId - The version id
* @param {String} value - The value of the key
* @return {number} - indicates if iteration should continue
*/
addContents(keyVersionSuffix, value) {
addContents(key, versionId, value) {
if (this._reachedMaxKeys()) {
return FILTER_END;
}
Expand All @@ -178,28 +110,23 @@ class DelimiterNonCurrent extends Delimiter {
}
++this.evaluatedKeys;

const { key, versionId } = this.parseKey(keyVersionSuffix);

this.NextKeyMarker = key;
this.NextVersionIdMarker = versionId;
this.nextKeyMarker = key;
this.nextVersionIdMarker = versionId;

// The master key serves two purposes:
// - It retrieves the expiration date for the previous version that is no longer current.
// - It excludes the current version from the list.
const isMasterKey = versionId === undefined;
const isMasterKey = this.masterKey === key && this.masterVersionId === versionId;
if (isMasterKey) {
this.masterKey = key;
this.masterVersionId = Version.from(value).getVersionId() || 'null';

this.staleDate = this.getLastModified(value);
return FILTER_ACCEPT;
}

const isCurrentVersion = this.masterKey === key && this.masterVersionId === versionId;
if (isCurrentVersion) {
// filter out the master version
return FILTER_ACCEPT;
}
// const isCurrentVersion = this.masterKey === key && this.masterVersionId === versionId;
// if (isCurrentVersion) {
// // filter out the master version
// return FILTER_ACCEPT;
// }

// The following version is pushed only:
// - if the "stale date" (picked up from the previous version) is available (JSON.parse has not failed),
Expand Down Expand Up @@ -262,17 +189,14 @@ class DelimiterNonCurrent extends Delimiter {
}

result() {
const result = {
Contents: this.Contents,
IsTruncated: this.IsTruncated,
};

if (this.IsTruncated) {
result.NextKeyMarker = this.NextKeyMarker;
result.NextVersionIdMarker = this.NextVersionIdMarker;
}
const { Versions, IsTruncated, NextKeyMarker, NextVersionIdMarker } = super.result();

return result;
return {
Contents: Versions,
IsTruncated,
NextKeyMarker,
NextVersionIdMarker,
};
}
}
module.exports = { DelimiterNonCurrent };

0 comments on commit a5032ee

Please sign in to comment.