diff --git a/.github/actions/archive-artifacts/action.yaml b/.github/actions/archive-artifacts/action.yaml index 0f18c7e653..47aad4a4fc 100644 --- a/.github/actions/archive-artifacts/action.yaml +++ b/.github/actions/archive-artifacts/action.yaml @@ -42,3 +42,21 @@ runs: sh -c "kubectl exec -i -n ${NAMESPACE} kcat -- \ kcat -L -b ${KAFKA_SERVICE} -t {} -C -o beginning -e -q -J \ > /tmp/artifacts/data/${STAGE}/kafka-messages-{}.log" + - name: Dump MongoDB + shell: bash + continue-on-error: true + run: |- + set -exu + + ZENKO_MONGODB_DATABASE="${ZENKO_MONGODB_DATABASE:-zenko-database}" + MONGODB_ROOT_USERNAME="${MONGODB_ROOT_USERNAME:-root}" + MONGODB_ROOT_PASSWORD="${MONGODB_ROOT_PASSWORD:-rootpass}" + NAMESPACE="${NAMESPACE:-default}" + DUMP_DIR="/tmp/mongodb.dump" + + kubectl exec -n ${NAMESPACE} data-db-mongodb-sharded-mongos-0 -- mongodump --db ${ZENKO_MONGODB_DATABASE} -u ${MONGODB_ROOT_USERNAME} -p ${MONGODB_ROOT_PASSWORD} --authenticationDatabase admin --out ${DUMP_DIR} + + kubectl exec -n ${NAMESPACE} data-db-mongodb-sharded-mongos-0 -- bash -c "for bson_file in ${DUMP_DIR}/${ZENKO_MONGODB_DATABASE}/*.bson; do json_file=\"${DUMP_DIR}/\$(basename \${bson_file} .bson).json\"; bsondump --outFile \${json_file} \${bson_file}; done" + + mkdir -p /tmp/artifacts/data/${STAGE}/mongodb-dump + kubectl cp ${NAMESPACE}/data-db-mongodb-sharded-mongos-0:${DUMP_DIR} /tmp/artifacts/data/${STAGE}/mongodb-dump diff --git a/.github/scripts/end2end/configs/zenko_dr_sink.yaml b/.github/scripts/end2end/configs/zenko_dr_sink.yaml index 1fb9ea103c..bf4c923555 100644 --- a/.github/scripts/end2end/configs/zenko_dr_sink.yaml +++ b/.github/scripts/end2end/configs/zenko_dr_sink.yaml @@ -11,7 +11,7 @@ spec: userSecretName: mongodb-db-creds-pra usernameKey: mongodb-username passwordKey: mongodb-password - databaseName: "pradb" + databaseName: pradb writeConcern: "majority" kafka: managed: diff --git a/.github/scripts/end2end/deploy-zenko.sh b/.github/scripts/end2end/deploy-zenko.sh index c151b2be79..3be48d0316 100755 --- a/.github/scripts/end2end/deploy-zenko.sh +++ b/.github/scripts/end2end/deploy-zenko.sh @@ -42,7 +42,7 @@ export ZENKO_ANNOTATIONS="annotations:" export ZENKO_MONGODB_ENDPOINT="data-db-mongodb-sharded.default.svc.cluster.local:27017" export ZENKO_MONGODB_CONFIG="writeConcern: 'majority' enableSharding: true" -export ZENKO_MONGODB_DATABASE="${ZENKO_MONGODB_DATABASE:-'datadb'}" +export ZENKO_MONGODB_DATABASE="${ZENKO_MONGODB_DATABASE:-datadb}" if [ "${TIME_PROGRESSION_FACTOR}" -gt 1 ]; then export ZENKO_ANNOTATIONS="$ZENKO_ANNOTATIONS diff --git a/.github/scripts/end2end/install-kind-dependencies.sh b/.github/scripts/end2end/install-kind-dependencies.sh index f8532e8731..ee568df9d5 100755 --- a/.github/scripts/end2end/install-kind-dependencies.sh +++ b/.github/scripts/end2end/install-kind-dependencies.sh @@ -21,7 +21,7 @@ MONGODB_ROOT_USERNAME=root MONGODB_ROOT_PASSWORD=rootpass MONGODB_APP_USERNAME=data MONGODB_APP_PASSWORD=datapass -MONGODB_APP_DATABASE="${ZENKO_MONGODB_DATABASE:-'datadb'}" +MONGODB_APP_DATABASE=${ZENKO_MONGODB_DATABASE:-datadb} MONGODB_RS_KEY=0123456789abcdef ENABLE_KEYCLOAK_HTTPS=${ENABLE_KEYCLOAK_HTTPS:-'false'} diff --git a/.github/scripts/end2end/prepare-pra.sh b/.github/scripts/end2end/prepare-pra.sh index f2b82a203c..bd49bf5296 100644 --- a/.github/scripts/end2end/prepare-pra.sh +++ b/.github/scripts/end2end/prepare-pra.sh @@ -6,7 +6,7 @@ export MONGODB_PRA_DATABASE="${MONGODB_PRA_DATABASE:-'pradb'}" export ZENKO_MONGODB_DATABASE="${MONGODB_PRA_DATABASE}" export ZENKO_MONGODB_SECRET_NAME="mongodb-db-creds-pra" -echo 'ZENKO_MONGODB_DATABASE="pradb"' >> "$GITHUB_ENV" +echo 'ZENKO_MONGODB_DATABASE=pradb' >> "$GITHUB_ENV" echo 'ZENKO_MONGODB_SECRET_NAME="mongodb-db-creds-pra"' >> "$GITHUB_ENV" echo 'ZENKO_IAM_INGRESS="iam.dr.zenko.local"' >> "$GITHUB_ENV" diff --git a/.github/workflows/end2end.yaml b/.github/workflows/end2end.yaml index e2a13ddeb4..c937f18dd1 100644 --- a/.github/workflows/end2end.yaml +++ b/.github/workflows/end2end.yaml @@ -479,7 +479,7 @@ jobs: - name: Deploy second Zenko for PRA run: bash deploy-zenko.sh end2end-pra default './configs/zenko.yaml' env: - ZENKO_MONGODB_DATABASE: "pradb" + ZENKO_MONGODB_DATABASE: pradb working-directory: ./.github/scripts/end2end - name: Add Keycloak pra user and assign StorageManager role shell: bash diff --git a/tests/ctst/common/common.ts b/tests/ctst/common/common.ts index 5c991ab239..2b3f40998d 100644 --- a/tests/ctst/common/common.ts +++ b/tests/ctst/common/common.ts @@ -5,7 +5,7 @@ import Zenko from 'world/Zenko'; import { safeJsonParse } from './utils'; import assert from 'assert'; import { Admin, Kafka } from 'kafkajs'; -import { +import { createBucketWithConfiguration, putObject, runActionAgainstBucket, @@ -15,6 +15,7 @@ import { addTransitionWorkflow, } from 'steps/utils/utils'; import { ActionPermissionsType } from 'steps/bucket-policies/utils'; +import constants from './constants'; setDefaultTimeout(Constants.DEFAULT_TIMEOUT); @@ -31,13 +32,21 @@ export async function cleanS3Bucket( if (!bucketName) { return; } + if (world.getSaved('objectLockMode') === constants.complianceRetention) { + // Do not try to clean a bucket with compliance retention + return; + } + Identity.useIdentity(IdentityEnum.ACCOUNT, world.getSaved('accountName') || + world.parameters.AccountName); world.resetCommand(); world.addCommandParameter({ bucket: bucketName }); - const createdObjects = world.getSaved>('createdObjects'); + const createdObjects = world.getCreatedObjects(); if (createdObjects !== undefined) { const results = await S3.listObjectVersions(world.getCommandParameters()); const res = safeJsonParse(results.stdout); - assert(res.ok); + if (!res.ok) { + throw results; + } const versions = res.result!.Versions || []; const deleteMarkers = res.result!.DeleteMarkers || []; await Promise.all(versions.concat(deleteMarkers).map(obj => { @@ -63,19 +72,14 @@ async function addMultipleObjects(this: Zenko, numberObjects: number, this.addToSaved('objectSize', sizeBytes); } if (userMD) { - this.addCommandParameter({ metadata: JSON.stringify(userMD) }); + this.addToSaved('userMetadata', userMD); } - this.addToSaved('objectName', objectNameFinal); - this.logger.debug('Adding object', { objectName: objectNameFinal }); lastResult = await putObject(this, objectNameFinal); - const createdObjects = this.getSaved>('createdObjects') || new Map(); - createdObjects.set(this.getSaved('objectName'), this.getSaved('versionId')); - this.addToSaved('createdObjects', createdObjects); } return lastResult; } -async function addUserMetadataToObject(this: Zenko, objectName: string|undefined, userMD: string) { +async function addUserMetadataToObject(this: Zenko, objectName: string | undefined, userMD: string) { const objName = objectName || this.getSaved('objectName'); const bucketName = this.getSaved('bucketName'); this.resetCommand(); @@ -154,7 +158,7 @@ Given('a tag on object {string} with key {string} and value {string}', this.resetCommand(); this.addCommandParameter({ bucket: this.getSaved('bucketName') }); this.addCommandParameter({ key: objectName }); - const versionId = this.getSaved>('createdObjects')?.get(objectName); + const versionId = this.getLatestObjectVersion(objectName); if (versionId) { this.addCommandParameter({ versionId }); } @@ -173,12 +177,12 @@ Then('object {string} should have the tag {string} with value {string}', this.resetCommand(); this.addCommandParameter({ bucket: this.getSaved('bucketName') }); this.addCommandParameter({ key: objectName }); - const versionId = this.getSaved>('createdObjects')?.get(objectName); + const versionId = this.getLatestObjectVersion(objectName); if (versionId) { this.addCommandParameter({ versionId }); } await S3.getObjectTagging(this.getCommandParameters()).then(res => { - const parsed = safeJsonParse<{ TagSet: [{Key: string, Value: string}] | undefined }>(res.stdout); + const parsed = safeJsonParse<{ TagSet: [{ Key: string, Value: string }] | undefined }>(res.stdout); assert(parsed.result!.TagSet?.some(tag => tag.Key === tagKey && tag.Value === tagValue)); }); }); @@ -188,14 +192,14 @@ Then('object {string} should have the user metadata with key {string} and value this.resetCommand(); this.addCommandParameter({ bucket: this.getSaved('bucketName') }); this.addCommandParameter({ key: objectName }); - const versionId = this.getSaved>('createdObjects')?.get(objectName); + const versionId = this.getLatestObjectVersion(objectName); if (versionId) { this.addCommandParameter({ versionId }); } const res = await S3.headObject(this.getCommandParameters()); assert.ifError(res.stderr); assert(res.stdout); - const parsed = safeJsonParse<{ Metadata: {[key: string]: string} | undefined }>(res.stdout); + const parsed = safeJsonParse<{ Metadata: { [key: string]: string } | undefined }>(res.stdout); assert(parsed.ok); assert(parsed.result!.Metadata); assert(parsed.result!.Metadata[userMDKey]); @@ -220,7 +224,7 @@ When('i delete object {string}', async function (this: Zenko, objectName: string this.resetCommand(); this.addCommandParameter({ bucket: this.getSaved('bucketName') }); this.addCommandParameter({ key: objName }); - const versionId = this.getSaved>('createdObjects')?.get(objName); + const versionId = this.getLatestObjectVersion(objName); if (versionId) { this.addCommandParameter({ versionId }); } @@ -251,7 +255,7 @@ Then('kafka consumed messages should not take too much place on disk', { timeout const kafkaAdmin = new Kafka({ brokers: [this.parameters.KafkaHosts] }).admin(); const topics: string[] = (await kafkaAdmin.listTopics()) .filter(t => (t.includes(this.parameters.InstanceID) && - !ignoredTopics.some(e => t.includes(e)))); + !ignoredTopics.some(e => t.includes(e)))); const previousOffsets = await getTopicsOffsets(topics, kafkaAdmin); @@ -378,12 +382,7 @@ Given('an upload size of {int} B for the object {string}', async function ( ) { this.addToSaved('objectSize', size); if (this.getSaved('preExistingObject')) { - if (objectName) { - this.addToSaved('objectName', objectName); - } else { - this.addToSaved('objectName', `object-${Utils.randomString()}`); - } - await putObject(this, this.getSaved('objectName')); + await putObject(this, objectName); } }); @@ -391,8 +390,7 @@ When('I PUT an object with size {int}', async function (this: Zenko, size: numbe if (size > 0) { this.addToSaved('objectSize', size); } - this.addToSaved('objectName', `object-${Utils.randomString()}`); const result = await addMultipleObjects.call( - this, 1, this.getSaved('objectName'), size); + this, 1, `object-${Utils.randomString()}`, size); this.setResult(result!); }); diff --git a/tests/ctst/common/constants.ts b/tests/ctst/common/constants.ts new file mode 100644 index 0000000000..d6d07f45b4 --- /dev/null +++ b/tests/ctst/common/constants.ts @@ -0,0 +1,4 @@ +export default { + complianceRetention: 'COMPLIANCE', + governanceRetention: 'GOVERNANCE', +}; diff --git a/tests/ctst/common/hooks.ts b/tests/ctst/common/hooks.ts index b36692e446..35c54c3a55 100644 --- a/tests/ctst/common/hooks.ts +++ b/tests/ctst/common/hooks.ts @@ -10,6 +10,9 @@ import { prepareQuotaScenarios, teardownQuotaScenarios } from 'steps/quotas/quot import { cleanS3Bucket } from './common'; import { cleanAzureContainer, cleanZenkoLocation } from 'steps/azureArchive'; import { displayDebuggingInformation, preparePRA } from 'steps/pra'; +import { + cleanupAccount, +} from './utils'; // HTTPS should not cause any error for CTST process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'; @@ -39,15 +42,24 @@ Before({ tags: '@Quotas', timeout: 1200000 }, async function (scenarioOptions) { await prepareQuotaScenarios(this as Zenko, scenarioOptions); }); +After(async function (this: Zenko, results) { + if (results.result?.status === 'FAILED') { + this.logger.warn('bucket was not cleaned for test', { + bucket: this.getSaved('bucketName'), + }); + return; + } + await cleanS3Bucket( + this, + this.getSaved('bucketName'), + ); +}); + After({ tags: '@Quotas' }, async function () { await teardownQuotaScenarios(this as Zenko); }); After({ tags: '@AzureArchive' }, async function (this: Zenko) { - await cleanS3Bucket( - this, - this.getSaved('bucketName'), - ); await cleanZenkoLocation( this, this.getSaved('locationName'), @@ -58,4 +70,16 @@ After({ tags: '@AzureArchive' }, async function (this: Zenko) { ); }); +After({ tags: '@BP-ASSUME_ROLE_USER_CROSS_ACCOUNT'}, async function (this: Zenko, results) { + const crossAccountName = this.getSaved('crossAccountName'); + + if (results.result?.status === 'FAILED' || !crossAccountName) { + this.logger.warn('cross account was not cleaned for test', { + crossAccountName, + }); + return; + } + await cleanupAccount(this, crossAccountName); +}); + export default Zenko; diff --git a/tests/ctst/common/utils.ts b/tests/ctst/common/utils.ts index 407e33cc62..a6dfee6fda 100644 --- a/tests/ctst/common/utils.ts +++ b/tests/ctst/common/utils.ts @@ -1,9 +1,16 @@ import { exec } from 'child_process'; import http from 'http'; import { createHash } from 'crypto'; +import { Command, IAM, Identity, IdentityEnum } from 'cli-testing'; import { - Command, -} from 'cli-testing'; + AttachedPolicy, + Group, + Policy, + Role, + User, +} from '@aws-sdk/client-iam'; +import { AWSCliOptions } from 'cli-testing'; +import Zenko from 'world/Zenko'; /** * This helper will dynamically extract a property from a CLI result @@ -140,3 +147,153 @@ export async function request(options: http.RequestOptions, data: string | undef export function hashStringAndKeepFirst20Characters(input: string) { return createHash('sha256').update(input).digest('hex').slice(0, 20); } + +export async function listAllEntities( + listFn: (params: AWSCliOptions) => Promise, + responseKey: string, +): Promise { + let marker; + const allEntities: T[] = []; + let parsedResponse; + do { + const response = await listFn({ marker }); + if (response.err) { + throw new Error(response.err); + } + parsedResponse = JSON.parse(response.stdout); + const entities = parsedResponse[responseKey] || []; + entities.forEach((entity: T) => { + if (entity.Path?.includes('/scality-internal/')) { + return; + } + allEntities.push(entity); + }); + marker = parsedResponse.Marker; + } while (parsedResponse.IsTruncated); + return allEntities; +}; + +export async function listAttachedPolicies( + listFn: (params: AWSCliOptions) => Promise, +): Promise { + let marker; + const allPolicies: T[] = []; + let parsedResponse; + do { + const response = await listFn({ marker }); + if (response.err) { + throw new Error(response.err); + } + parsedResponse = JSON.parse(response.stdout); + const policies = parsedResponse.AttachedPolicies || []; + policies.forEach((policy: T) => { + if (policy.PolicyArn?.includes('/scality-internal/')) { + return; + } + allPolicies.push(policy); + }); + marker = parsedResponse.Marker; + } while (parsedResponse.IsTruncated); + return allPolicies; +} + +export async function cleanupAccount(world: Zenko, accountName: string) { + try { + await world.deleteAccount(accountName); + } catch (err) { + world.logger?.debug('Account has attached resources',{ + accountName, + err, + }); + } + + try { + Identity.useIdentity(IdentityEnum.ACCOUNT, accountName); + + // List and detach policies for each user + const allUsers = await listAllEntities(IAM.listUsers, 'Users'); + for (const user of allUsers) { + const allUserPolicies = await listAttachedPolicies( + params => IAM.listAttachedUserPolicies({ userName: user.UserName, ...params }), + ); + for (const policy of allUserPolicies) { + const result = await IAM.detachUserPolicy({ + userName: user.UserName, policyArn: policy.PolicyArn }); + if (result.err) { + throw new Error(result.err); + } + } + } + + // List and detach policies for each group + const allGroups = await listAllEntities(IAM.listGroups, 'Groups'); + for (const group of allGroups) { + const allGroupPolicies = await listAttachedPolicies( + params => IAM.listAttachedGroupPolicies({ groupName: group.GroupName, ...params }), + ); + for (const policy of allGroupPolicies) { + const result = await IAM.detachGroupPolicy({ + groupName: group.GroupName, policyArn: policy.PolicyArn }); + if (result.err) { + throw new Error(result.err); + } + } + } + + // List and detach policies for each role + const allRoles = await listAllEntities(IAM.listRoles, 'Roles'); + for (const role of allRoles) { + const allRolePolicies = await listAttachedPolicies( + params => IAM.listAttachedRolePolicies({ roleName: role.RoleName, ...params }), + ); + for (const policy of allRolePolicies) { + const result = await IAM.detachRolePolicy({ + roleName: role.RoleName, policyArn: policy.PolicyArn }); + if (result.err) { + throw new Error(result.err); + } + } + } + + // Delete all policies + const allPolicies = await listAllEntities(IAM.listPolicies, 'Policies'); + for (const policy of allPolicies) { + const result = await IAM.deletePolicy({ policyArn: policy.Arn }); + if (result.err) { + throw new Error(result.err); + } + } + + // Delete all roles + for (const role of allRoles) { + const result = await IAM.deleteRole({ roleName: role.RoleName }); + if (result.err) { + throw new Error(result.err); + } + } + + // Delete all groups + for (const group of allGroups) { + const result = await IAM.deleteGroup({ groupName: group.GroupName }); + if (result.err) { + throw new Error(result.err); + } + } + + // Delete all users + for (const user of allUsers) { + const result = await IAM.deleteUser({ userName: user.UserName }); + if (result.err) { + throw new Error(result.err); + } + } + + // Finally, delete the account + await world.deleteAccount(accountName); + } catch (err) { + world.logger.warn('Error while deleting cross account', { + accountName, + error: err, + }); + } +} diff --git a/tests/ctst/steps/azureArchive.ts b/tests/ctst/steps/azureArchive.ts index 88be489a9d..0ee8fc4c62 100644 --- a/tests/ctst/steps/azureArchive.ts +++ b/tests/ctst/steps/azureArchive.ts @@ -158,7 +158,7 @@ export async function cleanAzureContainer( world: Zenko, bucketName: string, ): Promise { - const createdObjects = world.getSaved>('createdObjects'); + const createdObjects = world.getCreatedObjects(); if (!createdObjects) { return; } @@ -250,7 +250,7 @@ Then('object {string} should have the same data', async function (this: Zenko, o this.resetCommand(); this.addCommandParameter({ bucket: this.getSaved('bucketName') }); this.addCommandParameter({ key: objName }); - const versionId = this.getSaved>('createdObjects')?.get(objName); + const versionId = this.getLatestObjectVersion(objName); if (versionId) { this.addCommandParameter({ versionId }); } @@ -333,7 +333,7 @@ Then('the storage class of object {string} must stay {string} for {int} seconds' this.resetCommand(); this.addCommandParameter({ bucket: this.getSaved('bucketName') }); this.addCommandParameter({ key: objName }); - const versionId = this.getSaved>('createdObjects')?.get(objName); + const versionId = this.getLatestObjectVersion(objName); if (versionId) { this.addCommandParameter({ versionId }); } @@ -382,7 +382,7 @@ Then('object {string} should expire in {int} days', async function (this: Zenko, this.resetCommand(); this.addCommandParameter({ bucket: this.getSaved('bucketName') }); this.addCommandParameter({ key: objName }); - const versionId = this.getSaved>('createdObjects')?.get(objName); + const versionId = this.getLatestObjectVersion(objName); if (versionId) { this.addCommandParameter({ versionId }); } diff --git a/tests/ctst/steps/bucket-policies/common.ts b/tests/ctst/steps/bucket-policies/common.ts index c9aac94364..869ccdf514 100644 --- a/tests/ctst/steps/bucket-policies/common.ts +++ b/tests/ctst/steps/bucket-policies/common.ts @@ -52,8 +52,7 @@ Given('an existing bucket prepared for the action', async function (this: Zenko) this.getSaved('withObjectLock'), this.getSaved('retentionMode')); if (this.getSaved('preExistingObject')) { - this.addToSaved('objectName', `objectforbptests-${Utils.randomString()}`); - await putObject(this, this.getSaved('objectName')); + await putObject(this, `objectforbptests-${Utils.randomString()}`); } }); diff --git a/tests/ctst/steps/cloudserverAuth.ts b/tests/ctst/steps/cloudserverAuth.ts index b4ef37dc98..a75a104be4 100644 --- a/tests/ctst/steps/cloudserverAuth.ts +++ b/tests/ctst/steps/cloudserverAuth.ts @@ -21,13 +21,13 @@ When('the user tries to perform DeleteObjects', async function (this: Zenko) { this.resetCommand(); this.useSavedIdentity(); this.addCommandParameter({ bucket: this.getSaved('bucketName') }); - const objectNames = this.getSaved('objectNameArray'); + const objects = this.getCreatedObjects(); const param: { Objects: { Key: string }[] } = { Objects: [], }; - objectNames.forEach((objectName: string) => { - param.Objects.push({ Key: objectName }); - }); + for (const key of objects) { + param.Objects.push({ Key: key[0] }); + } this.addCommandParameter({ delete: JSON.stringify(param) }); this.setResult(await S3.deleteObjects(this.getCommandParameters())); }); diff --git a/tests/ctst/steps/notifications.ts b/tests/ctst/steps/notifications.ts index 414756dfc9..e2c802f7ef 100644 --- a/tests/ctst/steps/notifications.ts +++ b/tests/ctst/steps/notifications.ts @@ -1,9 +1,9 @@ -import { Then, Given, When, After } from '@cucumber/cucumber'; +import { Then, Given, When } from '@cucumber/cucumber'; import { strict as assert } from 'assert'; import { S3, Utils, KafkaHelper, AWSVersionObject, NotificationDestination } from 'cli-testing'; import { Message } from 'node-rdkafka'; -import { cleanS3Bucket } from 'common/common'; import Zenko from 'world/Zenko'; +import { putObject } from './utils/utils'; const KAFKA_TESTS_TIMEOUT = Number(process.env.KAFKA_TESTS_TIMEOUT) || 60000; @@ -38,16 +38,8 @@ interface QueueConfiguration { Events: string[]; } -async function putObject(world: Zenko) { - world.resetCommand(); - world.addCommandParameter({ bucket: world.getSaved('bucketName') }); - world.addCommandParameter({ key: world.getSaved('objectName') }); - const result = await S3.putObject(world.getCommandParameters()); - world.setResult(result); -} - -async function copyObject(world: Zenko) { - await putObject(world); +async function copyObject(world: Zenko, sourceObject: string) { + await putObject(world, sourceObject); world.resetCommand(); let objName = `object-${Utils.randomString()}`.toLocaleLowerCase(); if (world.getSaved('filterType')) { @@ -59,17 +51,17 @@ async function copyObject(world: Zenko) { world.addCommandParameter({ key: objName }); world.addCommandParameter({ copySource: - `${world.getSaved('bucketName')}/${world.getSaved('objectName') }`, + `${world.getSaved('bucketName')}/${sourceObject}`, }); world.addToSaved('objectName', objName); await S3.copyObject(world.getCommandParameters()); } -async function deleteObject(world: Zenko, putDeleteMarker = false) { - await putObject(world); +async function deleteObject(world: Zenko, objName: string, putDeleteMarker = false) { + await putObject(world, objName); world.resetCommand(); world.addCommandParameter({ bucket: world.getSaved('bucketName') }); - world.addCommandParameter({ key: world.getSaved('objectName') }); + world.addCommandParameter({ key: objName }); if (world.getSaved('bucketVersioning') !== 'Non versioned' && !putDeleteMarker) { const putResult = world.getResult(); const versionId = @@ -79,8 +71,8 @@ async function deleteObject(world: Zenko, putDeleteMarker = false) { await S3.deleteObject(world.getCommandParameters()); } -async function putTag(world: Zenko) { - await putObject(world); +async function putTag(world: Zenko, objName: string) { + await putObject(world, objName); world.resetCommand(); const tags = JSON.stringify({ TagSet: [{ @@ -89,24 +81,24 @@ async function putTag(world: Zenko) { }], }); world.addCommandParameter({ bucket: world.getSaved('bucketName') }); - world.addCommandParameter({ key: world.getSaved('objectName') }); + world.addCommandParameter({ key: objName }); world.addCommandParameter({ tagging: `'${tags}'` }); await S3.putObjectTagging(world.getCommandParameters()); } -async function deleteTag(world: Zenko) { - await putTag(world); +async function deleteTag(world: Zenko, objName: string) { + await putTag(world, objName); world.resetCommand(); world.addCommandParameter({ bucket: world.getSaved('bucketName') }); - world.addCommandParameter({ key: world.getSaved('objectName') }); + world.addCommandParameter({ key: objName }); await S3.deleteObjectTagging(world.getCommandParameters()); } -async function putAcl(world: Zenko) { - await putObject(world); +async function putAcl(world: Zenko, objName: string) { + await putObject(world, objName); world.resetCommand(); world.addCommandParameter({ bucket: world.getSaved('bucketName') }); - world.addCommandParameter({ key: world.getSaved('objectName') }); + world.addCommandParameter({ key: objName }); world.addCommandParameter({ acl: 'public-read' }); await S3.putObjectAcl(world.getCommandParameters()); } @@ -265,25 +257,25 @@ When('a {string} event is triggered {string} {string}', this.addToSaved('objectName', objName); switch (notificationType) { case 's3:ObjectCreated:Put': - await putObject(this); + await putObject(this, objName); break; case 's3:ObjectCreated:Copy': - await copyObject(this); + await copyObject(this, objName); break; case 's3:ObjectRemoved:Delete': - await deleteObject(this); + await deleteObject(this, objName); break; case 's3:ObjectTagging:Put': - await putTag(this); + await putTag(this, objName); break; case 's3:ObjectTagging:Delete': - await deleteTag(this); + await deleteTag(this, objName); break; case 's3:ObjectAcl:Put': - await putAcl(this); + await putAcl(this, objName); break; case 's3:ObjectRemoved:DeleteMarkerCreated': - await deleteObject(this, true); + await deleteObject(this, objName, true); break; default: break; @@ -304,7 +296,6 @@ Then('notifications should be enabled for {string} event in destination {int}', (this.getSaved('notificationDestinations')[destination]).destinationName; }) as QueueConfiguration; assert(destinationConfiguration.Events.includes(notificationType)); - await S3.deleteBucket(this.getCommandParameters()); }); Then('i should {string} a notification for {string} event in destination {int}', @@ -336,9 +327,3 @@ Then('i should {string} a notification for {string} event in destination {int}', assert.strictEqual(receivedNotification, expected); }); -After({ tags: '@BucketNotification' }, async function (this: Zenko) { - await cleanS3Bucket( - this, - this.getSaved('bucketName'), - ); -}); diff --git a/tests/ctst/steps/pra.ts b/tests/ctst/steps/pra.ts index 3e05debddc..88f13b2a0c 100644 --- a/tests/ctst/steps/pra.ts +++ b/tests/ctst/steps/pra.ts @@ -9,10 +9,11 @@ import { getPVCFromLabel, } from './utils/kubernetes'; import { + putObject, restoreObject, verifyObjectLocation, } from 'steps/utils/utils'; -import { Constants, Identity, IdentityEnum, S3, SuperAdmin, Utils } from 'cli-testing'; +import { Constants, Identity, IdentityEnum, SuperAdmin, Utils } from 'cli-testing'; import { safeJsonParse } from 'common/utils'; import assert from 'assert'; import { EntityType } from 'world/Zenko'; @@ -298,10 +299,7 @@ When('the DATA_ACCESSOR user tries to perform PutObject on {string} site', { tim } } - this.addCommandParameter({ bucket: this.getSaved('bucketName') }); - this.addCommandParameter({ key: `${Utils.randomString()}` }); - - this.setResult(await S3.putObject(this.getCommandParameters())); + await putObject(this); }); const volumeTimeout = 60000; diff --git a/tests/ctst/steps/sosapi.ts b/tests/ctst/steps/sosapi.ts index ebc72577dd..bc4e223e24 100644 --- a/tests/ctst/steps/sosapi.ts +++ b/tests/ctst/steps/sosapi.ts @@ -71,6 +71,5 @@ Then('the request should be {string}', async function (this: Zenko, result: stri const decision = this.checkResults([this.getResult()]); assert.strictEqual(decision, result === 'accepted'); this.addCommandParameter({ bucket: this.getSaved('bucketName') }); - await S3.deleteBucket(this.getCommandParameters()); await deleteFile(this.getSaved('tempFileName')); }); diff --git a/tests/ctst/steps/utils/utils.ts b/tests/ctst/steps/utils/utils.ts index 4887d6a49b..5420c212dc 100644 --- a/tests/ctst/steps/utils/utils.ts +++ b/tests/ctst/steps/utils/utils.ts @@ -12,6 +12,7 @@ import { import { extractPropertyFromResults, s3FunctionExtraParams, safeJsonParse } from 'common/utils'; import Zenko from 'world/Zenko'; import assert from 'assert'; +import constants from 'common/constants'; enum AuthorizationType { ALLOW = 'Allow', @@ -46,6 +47,7 @@ async function uploadSetup(world: Zenko, action: string) { world.addCommandParameter({ body: world.getSaved('tempFileName') }); } } + async function uploadTeardown(world: Zenko, action: string) { if (action !== 'PutObject' && action !== 'UploadPart') { return; @@ -53,6 +55,7 @@ async function uploadTeardown(world: Zenko, action: string) { const objectSize = world.getSaved('objectSize') || 0; if (objectSize > 0) { await deleteFile(world.getSaved('tempFileName')); + world.deleteKeyFromCommand('body'); } } @@ -78,8 +81,8 @@ async function runActionAgainstBucket(world: Zenko, action: string) { world.resetCommand(); world.addToSaved('ifS3Standard', true); world.addCommandParameter({ bucket: world.getSaved('bucketName') }); - if (world.getSaved('versionId')) { - world.addCommandParameter({ versionId: world.getSaved('versionId') }); + if (world.getSaved('lastVersionId')) { + world.addCommandParameter({ versionId: world.getSaved('lastVersionId') }); } // if copy object, set copy source as the saved object name, and the key as a new object name if (action === 'CopyObject') { @@ -181,7 +184,8 @@ async function createBucketWithConfiguration( world.addCommandParameter({ versioningConfiguration: 'Status=Enabled' }); await S3.putBucketVersioning(world.getCommandParameters()); } - if (retentionMode === 'GOVERNANCE' || retentionMode === 'COMPLIANCE') { + if (retentionMode === constants.governanceRetention || retentionMode === constants.complianceRetention) { + world.addToSaved('objectLockMode', retentionMode); world.resetCommand(); world.addCommandParameter({ bucket: usedBucketName }); world.addCommandParameter({ @@ -196,18 +200,25 @@ async function createBucketWithConfiguration( } async function putObject(world: Zenko, objectName?: string) { - world.addToSaved('objectName', objectName || Utils.randomString()); - const objectNameArray = world.getSaved('objectNameArray') || []; - objectNameArray.push(world.getSaved('objectName')); - world.addToSaved('objectNameArray', objectNameArray); + world.resetCommand(); + let finalObjectName = objectName; + if (!finalObjectName) { + finalObjectName = `${Utils.randomString()}`; + } + world.addToSaved('objectName', finalObjectName); + world.logger.debug('Adding object', { objectName: finalObjectName }); await uploadSetup(world, 'PutObject'); - world.addCommandParameter({ key: world.getSaved('objectName') }); + world.addCommandParameter({ key: finalObjectName }); world.addCommandParameter({ bucket: world.getSaved('bucketName') }); + const userMetadata = world.getSaved('userMetadata'); + if (userMetadata) { + world.addCommandParameter({ metadata: JSON.stringify(userMetadata) }); + } const result = await S3.putObject(world.getCommandParameters()); - world.addToSaved('versionId', extractPropertyFromResults( - result, 'VersionId' - )); + const versionId = extractPropertyFromResults(result, 'VersionId'); + world.saveCreatedObject(finalObjectName, versionId || ''); await uploadTeardown(world, 'PutObject'); + world.setResult(result); return result; } @@ -292,7 +303,7 @@ async function verifyObjectLocation(this: Zenko, objectName: string, this.resetCommand(); this.addCommandParameter({ bucket: this.getSaved('bucketName') }); this.addCommandParameter({ key: objName }); - const versionId = this.getSaved>('createdObjects')?.get(objName); + const versionId = this.getLatestObjectVersion(objName); if (versionId) { this.addCommandParameter({ versionId }); } @@ -332,7 +343,7 @@ async function restoreObject(this: Zenko, objectName: string, days: number) { this.resetCommand(); this.addCommandParameter({ bucket: this.getSaved('bucketName') }); this.addCommandParameter({ key: objName }); - const versionId = this.getSaved>('createdObjects')?.get(objName); + const versionId = this.getLatestObjectVersion(objName); if (versionId) { this.addCommandParameter({ versionId }); } diff --git a/tests/ctst/world/Zenko.ts b/tests/ctst/world/Zenko.ts index 29db94300f..af2ecce340 100644 --- a/tests/ctst/world/Zenko.ts +++ b/tests/ctst/world/Zenko.ts @@ -445,6 +445,13 @@ export default class Zenko extends World { this.saveIdentityInformation(accountName, IdentityEnum.ACCOUNT, accountName); } + async deleteAccount(name: string) { + if (!name) { + throw new Error('No account name provided'); + } + await SuperAdmin.deleteAccount({ accountName: name }); + } + /** * Creates an assumed role session with a duration of 12 hours. * @param {boolean} crossAccount - If true, the role will be assumed cross account. @@ -479,6 +486,7 @@ export default class Zenko extends World { }); Identity.addIdentity(IdentityEnum.ACCOUNT, account2.account.name, account2Credentials, undefined, true); + this.addToSaved('crossAccountName', account2.account.name); accountToBeAssumedFrom = account2.account.name; } @@ -946,6 +954,25 @@ export default class Zenko extends World { return await this.managementAPIRequest('DELETE', `/config/${this.parameters.InstanceID}/location/${locationName}`); } + + saveCreatedObject(objectName: string, versionId: string) { + const createdObjects = this.getSaved>('createdObjects') || new Map(); + createdObjects.set(objectName, (createdObjects.get(objectName) || []).concat(versionId)); + this.addToSaved('createdObjects', createdObjects); + this.addToSaved('lastVersionId', versionId); + } + + getCreatedObjects() { + return this.getSaved>('createdObjects'); + } + + getCreatedObject(objectName: string) { + return this.getSaved>('createdObjects')?.get(objectName); + } + + getLatestObjectVersion(objectName: string) { + return this.getSaved>('createdObjects')?.get(objectName)?.slice(-1)[0]; + } } setWorldConstructor(Zenko);