From af7c4d83c7f5b61e907a90622c082a373f3995d9 Mon Sep 17 00:00:00 2001 From: Shilpa Jagannath Date: Mon, 30 Oct 2023 11:29:06 -0400 Subject: [PATCH 1/9] rgw/multisite: reconfigure checkpoint delay to 90s Signed-off-by: Shilpa Jagannath --- qa/suites/rgw/multisite/tasks/test_multi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/suites/rgw/multisite/tasks/test_multi.yaml b/qa/suites/rgw/multisite/tasks/test_multi.yaml index 1a65a67b5aa52..422535db6099c 100644 --- a/qa/suites/rgw/multisite/tasks/test_multi.yaml +++ b/qa/suites/rgw/multisite/tasks/test_multi.yaml @@ -14,4 +14,4 @@ tasks: - rgw-multisite: - rgw-multisite-tests: config: - reconfigure_delay: 60 + reconfigure_delay: 90 From 32fb2b923bc07b2228b8fe7d1c1f4bc2f3bf263b Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Tue, 31 Oct 2023 10:51:28 -0400 Subject: [PATCH 2/9] qa/rgw/multisite: reduce polling intervals from 20s to 5s Signed-off-by: Casey Bodley (cherry picked from commit bc084c0ac64477abdbf54d96bc4981edf41e1343) --- qa/suites/rgw/multisite/overrides.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qa/suites/rgw/multisite/overrides.yaml b/qa/suites/rgw/multisite/overrides.yaml index 13b498d167f60..49f95afb00f49 100644 --- a/qa/suites/rgw/multisite/overrides.yaml +++ b/qa/suites/rgw/multisite/overrides.yaml @@ -15,6 +15,8 @@ overrides: rgw curl low speed time: 300 rgw md log max shards: 4 rgw data log num shards: 4 + rgw data sync poll interval: 5 + rgw meta sync poll interval: 5 rgw sync obj etag verify: true rgw sync meta inject err probability: 0.1 rgw sync data inject err probability: 0.1 From a77c2d42277e44633643bcc1f8969f90724c49ef Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Tue, 31 Oct 2023 10:53:25 -0400 Subject: [PATCH 3/9] qa/rgw/multisite: double cluster nodes and osd count > WARNING: The OSD cluster is overloaded and struggling to complete ops. You need more capacity to serve this level of demand. give each cluster 2 nodes instead of 1, and 6 osds instead of 3 Signed-off-by: Casey Bodley (cherry picked from commit 9a80cf4bf473c95c8d0b18fcdce88be20f74f973) --- qa/suites/rgw/multisite/clusters.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/qa/suites/rgw/multisite/clusters.yaml b/qa/suites/rgw/multisite/clusters.yaml index 536ef7ca446e7..0250838023906 100644 --- a/qa/suites/rgw/multisite/clusters.yaml +++ b/qa/suites/rgw/multisite/clusters.yaml @@ -1,3 +1,5 @@ roles: -- [c1.mon.a, c1.mgr.x, c1.osd.0, c1.osd.1, c1.osd.2, c1.client.0, c1.client.1] -- [c2.mon.a, c2.mgr.x, c2.osd.0, c2.osd.1, c2.osd.2, c2.client.0, c2.client.1] +- [c1.mon.a, c1.osd.0, c1.osd.1, c1.osd.2, c1.client.0] +- [c1.mgr.x, c1.osd.3, c1.osd.4, c1.osd.5, c1.client.1] +- [c2.mon.a, c2.osd.0, c2.osd.1, c2.osd.2, c2.client.0] +- [c2.mgr.x, c2.osd.3, c2.osd.4, c2.osd.5, c2.client.1] From 627dd7b86d0c59768625b508b614fa35d008708b Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Thu, 9 Nov 2023 16:22:36 -0500 Subject: [PATCH 4/9] qa/rgw/multisite: add two-zone configuration Signed-off-by: Casey Bodley (cherry picked from commit 14204442ac2eaaaa3f40a76403da49f0f4ed92c1) --- qa/suites/rgw/multisite/realms/two-zones.yaml | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 qa/suites/rgw/multisite/realms/two-zones.yaml diff --git a/qa/suites/rgw/multisite/realms/two-zones.yaml b/qa/suites/rgw/multisite/realms/two-zones.yaml new file mode 100644 index 0000000000000..3564c9619b7cb --- /dev/null +++ b/qa/suites/rgw/multisite/realms/two-zones.yaml @@ -0,0 +1,21 @@ +overrides: + rgw-multisite: + realm: + name: test-realm + is default: true + zonegroups: + - name: test-zonegroup + is_master: true + is_default: true + endpoints: [c1.client.0] + enabled_features: ['resharding'] + zones: + - name: test-zone1 + is_master: true + is_default: true + endpoints: [c1.client.0] + - name: test-zone2 + is_default: true + endpoints: [c2.client.0] + rgw-multisite-tests: + args: [tests.py] From 79d520698d4dc3f06535d3c8fc7ad119fd52ada1 Mon Sep 17 00:00:00 2001 From: Shilpa Jagannath Date: Fri, 12 Apr 2024 16:46:11 -0400 Subject: [PATCH 5/9] qa/multisite: disable sync policy, endpoint and data_sync_init tests until fixed. Signed-off-by: Shilpa Jagannath --- src/test/rgw/rgw_multi/tests.py | 692 +++++++++++++++++--------------- 1 file changed, 366 insertions(+), 326 deletions(-) diff --git a/src/test/rgw/rgw_multi/tests.py b/src/test/rgw/rgw_multi/tests.py index 78c8d722b2ff7..33c885d239bfb 100644 --- a/src/test/rgw/rgw_multi/tests.py +++ b/src/test/rgw/rgw_multi/tests.py @@ -765,6 +765,38 @@ def check_oidc_providers_eq(zone_conn1, zone_conn2): eq(p1, p2) check_oidc_provider_eq(zone_conn1, zone_conn2, p1['Arn']) +def create_zone_bucket(zone): + b_name = gen_bucket_name() + log.info('create bucket zone=%s name=%s', zone.name, b_name) + bucket = zone.create_bucket(b_name) + return bucket + +def create_object(zone_conn, bucket, objname, content): + k = new_key(zone_conn, bucket.name, objname) + k.set_contents_from_string(content) + +def create_objects(zone_conn, bucket, obj_arr, content): + for objname in obj_arr: + create_object(zone_conn, bucket, objname, content) + +def check_object_exists(bucket, objname, content = None): + k = bucket.get_key(objname) + assert_not_equal(k, None) + if (content != None): + assert_equal(k.get_contents_as_string(encoding='ascii'), content) + +def check_objects_exist(bucket, obj_arr, content = None): + for objname in obj_arr: + check_object_exists(bucket, objname, content) + +def check_object_not_exists(bucket, objname): + k = bucket.get_key(objname) + assert_equal(k, None) + +def check_objects_not_exist(bucket, obj_arr): + for objname in obj_arr: + check_object_not_exists(bucket, objname) + def test_object_sync(): zonegroup = realm.master_zonegroup() zonegroup_conns = ZonegroupConns(zonegroup) @@ -1536,93 +1568,6 @@ def make_test_bucket(): cold_bilog = bilog_list(zone.zone, cold_bucket.name) assert(len(cold_bilog) == 0) -def test_bucket_reshard_index_log_trim(): - zonegroup = realm.master_zonegroup() - zonegroup_conns = ZonegroupConns(zonegroup) - - zone = zonegroup_conns.rw_zones[0] - - # create a test bucket, upload some objects, and wait for sync - def make_test_bucket(): - name = gen_bucket_name() - log.info('create bucket zone=%s name=%s', zone.name, name) - bucket = zone.conn.create_bucket(name) - for objname in ('a', 'b', 'c', 'd'): - k = new_key(zone, name, objname) - k.set_contents_from_string('foo') - zonegroup_meta_checkpoint(zonegroup) - zonegroup_bucket_checkpoint(zonegroup_conns, name) - return bucket - - # create a 'test' bucket - test_bucket = make_test_bucket() - - # checking bucket layout before resharding - json_obj_1 = bucket_layout(zone.zone, test_bucket.name) - assert(len(json_obj_1['layout']['logs']) == 1) - - first_gen = json_obj_1['layout']['current_index']['gen'] - - before_reshard_bilog = bilog_list(zone.zone, test_bucket.name, ['--gen', str(first_gen)]) - assert(len(before_reshard_bilog) == 4) - - # Resharding the bucket - zone.zone.cluster.admin(['bucket', 'reshard', - '--bucket', test_bucket.name, - '--num-shards', '3', - '--yes-i-really-mean-it']) - - # checking bucket layout after 1st resharding - json_obj_2 = bucket_layout(zone.zone, test_bucket.name) - assert(len(json_obj_2['layout']['logs']) == 2) - - second_gen = json_obj_2['layout']['current_index']['gen'] - - after_reshard_bilog = bilog_list(zone.zone, test_bucket.name, ['--gen', str(second_gen)]) - assert(len(after_reshard_bilog) == 0) - - # upload more objects - for objname in ('e', 'f', 'g', 'h'): - k = new_key(zone, test_bucket.name, objname) - k.set_contents_from_string('foo') - zonegroup_bucket_checkpoint(zonegroup_conns, test_bucket.name) - - # Resharding the bucket again - zone.zone.cluster.admin(['bucket', 'reshard', - '--bucket', test_bucket.name, - '--num-shards', '3', - '--yes-i-really-mean-it']) - - # checking bucket layout after 2nd resharding - json_obj_3 = bucket_layout(zone.zone, test_bucket.name) - assert(len(json_obj_3['layout']['logs']) == 3) - - zonegroup_bucket_checkpoint(zonegroup_conns, test_bucket.name) - - bilog_autotrim(zone.zone) - - # checking bucket layout after 1st bilog autotrim - json_obj_4 = bucket_layout(zone.zone, test_bucket.name) - assert(len(json_obj_4['layout']['logs']) == 2) - - bilog_autotrim(zone.zone) - - # checking bucket layout after 2nd bilog autotrim - json_obj_5 = bucket_layout(zone.zone, test_bucket.name) - assert(len(json_obj_5['layout']['logs']) == 1) - - bilog_autotrim(zone.zone) - - # upload more objects - for objname in ('i', 'j', 'k', 'l'): - k = new_key(zone, test_bucket.name, objname) - k.set_contents_from_string('foo') - zonegroup_bucket_checkpoint(zonegroup_conns, test_bucket.name) - - # verify the bucket has non-empty bilog - test_bilog = bilog_list(zone.zone, test_bucket.name) - assert(len(test_bilog) > 0) - @attr('bucket_reshard') def test_bucket_reshard_incremental(): zonegroup = realm.master_zonegroup() @@ -1725,46 +1670,341 @@ def write_most_shards(zone, bucket_name, num_shards): k = new_key(zone, bucket_name, obj) k.set_contents_from_string('foo') -def reshard_bucket(zone, bucket_name, num_shards): - """ - Reshard a bucket - """ - cmd = ['bucket', 'reshard'] + zone.zone_args() - cmd += ['--bucket', bucket_name] - cmd += ['--num-shards', str(num_shards)] - cmd += ['--yes-i-really-mean-it'] - zone.cluster.admin(cmd) +def reshard_bucket(zone, bucket_name, num_shards): + """ + Reshard a bucket + """ + cmd = ['bucket', 'reshard'] + zone.zone_args() + cmd += ['--bucket', bucket_name] + cmd += ['--num-shards', str(num_shards)] + cmd += ['--yes-i-really-mean-it'] + zone.cluster.admin(cmd) + +def get_obj_names(zone, bucket_name, maxobjs): + """ + Get names of objects in a bucket. + """ + cmd = ['bucket', 'list'] + zone.zone_args() + cmd += ['--bucket', bucket_name] + cmd += ['--max-entries', str(maxobjs)] + objs_json, _ = zone.cluster.admin(cmd, read_only=True) + objs = json.loads(objs_json) + return [o['name'] for o in objs] + +def bucket_keys_eq(zone1, zone2, bucket_name): + """ + Ensure that two buckets have the same keys, but get the lists through + radosgw-admin rather than S3 so it can be used when radosgw isn't running. + Only works for buckets of 10,000 objects since the tests calling it don't + need more, and the output from bucket list doesn't have an obvious marker + with which to continue. + """ + keys1 = get_obj_names(zone1, bucket_name, 10000) + keys2 = get_obj_names(zone2, bucket_name, 10000) + for key1, key2 in zip_longest(keys1, keys2): + if key1 is None: + log.critical('key=%s is missing from zone=%s', key1.name, + zone1.name) + assert False + if key2 is None: + log.critical('key=%s is missing from zone=%s', key2.name, + zone2.name) + assert False + +def test_role_sync(): + zonegroup = realm.master_zonegroup() + zonegroup_conns = ZonegroupConns(zonegroup) + roles, zone_role = create_role_per_zone(zonegroup_conns) + + zonegroup_meta_checkpoint(zonegroup) + + for source_conn, target_conn in combinations(zonegroup_conns.zones, 2): + if target_conn.zone.has_roles(): + check_roles_eq(source_conn, target_conn) + +def test_role_delete_sync(): + zonegroup = realm.master_zonegroup() + zonegroup_conns = ZonegroupConns(zonegroup) + role_name = gen_role_name() + log.info('create role zone=%s name=%s', zonegroup_conns.master_zone.name, role_name) + policy_document = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam:::user/testuser\"]},\"Action\":[\"sts:AssumeRole\"]}]}" + zonegroup_conns.master_zone.iam_conn.create_role(RoleName=role_name, AssumeRolePolicyDocument=policy_document) + + zonegroup_meta_checkpoint(zonegroup) + + for zone in zonegroup_conns.zones: + log.info(f'checking if zone: {zone.name} has role: {role_name}') + zone.iam_conn.get_role(RoleName=role_name) + log.info(f'success, zone: {zone.name} has role: {role_name}') + + log.info(f"deleting role: {role_name}") + zonegroup_conns.master_zone.iam_conn.delete_role(RoleName=role_name) + zonegroup_meta_checkpoint(zonegroup) + + for zone in zonegroup_conns.zones: + log.info(f'checking if zone: {zone.name} does not have role: {role_name}') + assert_raises(zone.iam_conn.exceptions.NoSuchEntityException, + zone.iam_conn.get_role, RoleName=role_name) + log.info(f'success, zone: {zone.name} does not have role: {role_name}') + +@attr('topic notification') +def test_topic_notification_sync(): + zonegroup = realm.master_zonegroup() + zonegroup_meta_checkpoint(zonegroup) + # let wait for users and other settings to sync across all zones. + time.sleep(config.checkpoint_delay) + # create topics in each zone. + zonegroup_conns = ZonegroupConns(zonegroup) + topic_arns, zone_topic = create_topic_per_zone(zonegroup_conns) + log.debug("topic_arns: %s", topic_arns) + + zonegroup_meta_checkpoint(zonegroup) + + # verify topics exists in all zones + for conn in zonegroup_conns.zones: + topic_list = conn.list_topics() + log.debug("topics for zone=%s = %s", conn.name, topic_list) + assert_equal(len(topic_list), len(topic_arns)) + for topic_arn_map in topic_list: + assert_true(topic_arn_map['TopicArn'] in topic_arns) + + # create a bucket + bucket = zonegroup_conns.rw_zones[0].create_bucket(gen_bucket_name()) + log.debug('created bucket=%s', bucket.name) + zonegroup_meta_checkpoint(zonegroup) + + # create bucket_notification in each zone. + notification_ids = [] + num = 1 + for zone_conn, topic_arn in zone_topic: + noti_id = "bn" + '-' + run_prefix + '-' + str(num) + notification_ids.append(noti_id) + topic_conf = {'Id': noti_id, + 'TopicArn': topic_arn, + 'Events': ['s3:ObjectCreated:*'] + } + num += 1 + log.info('creating bucket notification for zone=%s name=%s', zone_conn.name, noti_id) + zone_conn.create_notification(bucket.name, [topic_conf]) + zonegroup_meta_checkpoint(zonegroup) + + # verify notifications exists in all zones + for conn in zonegroup_conns.zones: + notification_list = conn.list_notifications(bucket.name) + log.debug("notifications for zone=%s = %s", conn.name, notification_list) + assert_equal(len(notification_list), len(topic_arns)) + for notification in notification_list: + assert_true(notification['Id'] in notification_ids) + + # verify bucket_topic mapping + # create a new bucket and subcribe it to first topic. + bucket_2 = zonegroup_conns.rw_zones[0].create_bucket(gen_bucket_name()) + notif_id = "bn-2" + '-' + run_prefix + topic_conf = {'Id': notif_id, + 'TopicArn': topic_arns[0], + 'Events': ['s3:ObjectCreated:*'] + } + zonegroup_conns.rw_zones[0].create_notification(bucket_2.name, [topic_conf]) + zonegroup_meta_checkpoint(zonegroup) + for conn in zonegroup_conns.zones: + topics = get_topics(conn.zone) + for topic in topics: + if topic['arn'] == topic_arns[0]: + assert_equal(len(topic['subscribed_buckets']), 2) + assert_true(bucket_2.name in topic['subscribed_buckets']) + else: + assert_equal(len(topic['subscribed_buckets']), 1) + assert_true(bucket.name in topic['subscribed_buckets']) + + # delete the 2nd bucket and verify the mapping is removed. + zonegroup_conns.rw_zones[0].delete_bucket(bucket_2.name) + zonegroup_meta_checkpoint(zonegroup) + for conn in zonegroup_conns.zones: + topics = get_topics(conn.zone) + for topic in topics: + assert_equal(len(topic['subscribed_buckets']), 1) + '''TODO(Remove the break once the https://tracker.ceph.com/issues/20802 + is fixed, as the secondary site bucket instance info is currently not + getting deleted coz of the bug hence the bucket-topic mapping + deletion is not invoked on secondary sites.)''' + break + + # delete notifications + zonegroup_conns.rw_zones[0].delete_notifications(bucket.name) + log.debug('Deleting all notifications for bucket=%s', bucket.name) + zonegroup_meta_checkpoint(zonegroup) + + # verify notification deleted in all zones + for conn in zonegroup_conns.zones: + notification_list = conn.list_notifications(bucket.name) + assert_equal(len(notification_list), 0) + + # delete topics + for zone_conn, topic_arn in zone_topic: + log.debug('deleting topic zone=%s arn=%s', zone_conn.name, topic_arn) + zone_conn.delete_topic(topic_arn) + zonegroup_meta_checkpoint(zonegroup) + + # verify topics deleted in all zones + for conn in zonegroup_conns.zones: + topic_list = conn.list_topics() + assert_equal(len(topic_list), 0) + +def test_account_metadata_sync(): + zonegroup = realm.master_zonegroup() + zonegroup_conns = ZonegroupConns(zonegroup) + + inline_policy = json.dumps({'Version': '2012-10-17', 'Statement': [{'Effect': 'Allow', 'Action': 's3:*', 'Resource': '*'}]}) + managed_policy_arn = 'arn:aws:iam::aws:policy/AmazonS3FullAccess' + + for source_conn in zonegroup_conns.rw_zones: + iam = source_conn.iam_conn + name = source_conn.name + # create user, add access key, user policy, managed policy + iam.create_user(UserName=name) + iam.create_access_key(UserName=name) + iam.put_user_policy(UserName=name, PolicyName='Allow', PolicyDocument=inline_policy) + iam.attach_user_policy(UserName=name, PolicyArn=managed_policy_arn) + # create group, group policy, managed policy, add user to group + iam.create_group(GroupName=name) + iam.put_group_policy(GroupName=name, PolicyName='Allow', PolicyDocument=inline_policy) + iam.attach_group_policy(GroupName=name, PolicyArn=managed_policy_arn) + iam.add_user_to_group(GroupName=name, UserName=name) + # create role, role policy, managed policy + iam.create_role(RoleName=name, AssumeRolePolicyDocument=json.dumps({'Version': '2012-10-17', 'Statement': [{'Effect': 'Allow', 'Principal': {'AWS': 'arn:aws:iam:::user/testuser'}, 'Action': ['sts:AssumeRole']}]})) + iam.put_role_policy(RoleName=name, PolicyName='Allow', PolicyDocument=inline_policy) + iam.attach_role_policy(RoleName=name, PolicyArn=managed_policy_arn) + # TODO: test oidc provider + #iam.create_open_id_connect_provider(ClientIDList=['clientid'], ThumbprintList=['3768084dfb3d2b68b7897bf5f565da8efEXAMPLE'], Url=f'http://{name}.example.com') + + realm_meta_checkpoint(realm) + + # check that all users/groups/roles are equal across all zones + for source_conn, target_conn in combinations(zonegroup_conns.zones, 2): + if target_conn.zone.has_roles(): + check_roles_eq(source_conn, target_conn) + check_users_eq(source_conn, target_conn) + check_groups_eq(source_conn, target_conn) + check_oidc_providers_eq(source_conn, target_conn) + + for source_conn in zonegroup_conns.rw_zones: + iam = source_conn.iam_conn + name = source_conn.name + + #iam.delete_open_id_connect_provider(OpenIDConnectProviderArn=f'arn:aws:iam::RGW11111111111111111:oidc-provider/{name}.example.com') + + iam.detach_role_policy(RoleName=name, PolicyArn=managed_policy_arn) + iam.delete_role_policy(RoleName=name, PolicyName='Allow') + iam.delete_role(RoleName=name) + + iam.remove_user_from_group(GroupName=name, UserName=name) + iam.detach_group_policy(GroupName=name, PolicyArn=managed_policy_arn) + iam.delete_group_policy(GroupName=name, PolicyName='Allow') + iam.delete_group(GroupName=name) + + iam.detach_user_policy(UserName=name, PolicyArn=managed_policy_arn) + iam.delete_user_policy(UserName=name, PolicyName='Allow') + key_id = iam.list_access_keys(UserName=name)['AccessKeyMetadata'][0]['AccessKeyId'] + iam.delete_access_key(UserName=name, AccessKeyId=key_id) + iam.delete_user(UserName=name) + + realm_meta_checkpoint(realm) + + # check that all users/groups/roles are equal across all zones + for source_conn, target_conn in combinations(zonegroup_conns.zones, 2): + if target_conn.zone.has_roles(): + check_roles_eq(source_conn, target_conn) + check_users_eq(source_conn, target_conn) + check_groups_eq(source_conn, target_conn) + check_oidc_providers_eq(source_conn, target_conn) + + +# TODO: move sync policy and endpoint tests into separate test file +# fix data sync init tests +''' +def test_bucket_reshard_index_log_trim(): + zonegroup = realm.master_zonegroup() + zonegroup_conns = ZonegroupConns(zonegroup) + + zone = zonegroup_conns.rw_zones[0] + + # create a test bucket, upload some objects, and wait for sync + def make_test_bucket(): + name = gen_bucket_name() + log.info('create bucket zone=%s name=%s', zone.name, name) + bucket = zone.conn.create_bucket(name) + for objname in ('a', 'b', 'c', 'd'): + k = new_key(zone, name, objname) + k.set_contents_from_string('foo') + zonegroup_meta_checkpoint(zonegroup) + zonegroup_bucket_checkpoint(zonegroup_conns, name) + return bucket + + # create a 'test' bucket + test_bucket = make_test_bucket() + + # checking bucket layout before resharding + json_obj_1 = bucket_layout(zone.zone, test_bucket.name) + assert(len(json_obj_1['layout']['logs']) == 1) + + first_gen = json_obj_1['layout']['current_index']['gen'] + + before_reshard_bilog = bilog_list(zone.zone, test_bucket.name, ['--gen', str(first_gen)]) + assert(len(before_reshard_bilog) == 4) + + # Resharding the bucket + zone.zone.cluster.admin(['bucket', 'reshard', + '--bucket', test_bucket.name, + '--num-shards', '13']) + + # checking bucket layout after 1st resharding + json_obj_2 = bucket_layout(zone.zone, test_bucket.name) + assert(len(json_obj_2['layout']['logs']) == 2) + + second_gen = json_obj_2['layout']['current_index']['gen'] + + after_reshard_bilog = bilog_list(zone.zone, test_bucket.name, ['--gen', str(second_gen)]) + assert(len(after_reshard_bilog) == 0) + + # upload more objects + for objname in ('e', 'f', 'g', 'h'): + k = new_key(zone, test_bucket.name, objname) + k.set_contents_from_string('foo') + zonegroup_bucket_checkpoint(zonegroup_conns, test_bucket.name) + + # Resharding the bucket again + zone.zone.cluster.admin(['bucket', 'reshard', + '--bucket', test_bucket.name, + '--num-shards', '15']) + + # checking bucket layout after 2nd resharding + json_obj_3 = bucket_layout(zone.zone, test_bucket.name) + assert(len(json_obj_3['layout']['logs']) == 3) + + zonegroup_bucket_checkpoint(zonegroup_conns, test_bucket.name) + + bilog_autotrim(zone.zone) + + # checking bucket layout after 1st bilog autotrim + json_obj_4 = bucket_layout(zone.zone, test_bucket.name) + assert(len(json_obj_4['layout']['logs']) == 2) + + bilog_autotrim(zone.zone) -def get_obj_names(zone, bucket_name, maxobjs): - """ - Get names of objects in a bucket. - """ - cmd = ['bucket', 'list'] + zone.zone_args() - cmd += ['--bucket', bucket_name] - cmd += ['--max-entries', str(maxobjs)] - objs_json, _ = zone.cluster.admin(cmd, read_only=True) - objs = json.loads(objs_json) - return [o['name'] for o in objs] + # checking bucket layout after 2nd bilog autotrim + json_obj_5 = bucket_layout(zone.zone, test_bucket.name) + time.sleep(config.checkpoint_delay) + assert(len(json_obj_5['layout']['logs']) == 1) -def bucket_keys_eq(zone1, zone2, bucket_name): - """ - Ensure that two buckets have the same keys, but get the lists through - radosgw-admin rather than S3 so it can be used when radosgw isn't running. - Only works for buckets of 10,000 objects since the tests calling it don't - need more, and the output from bucket list doesn't have an obvious marker - with which to continue. - """ - keys1 = get_obj_names(zone1, bucket_name, 10000) - keys2 = get_obj_names(zone2, bucket_name, 10000) - for key1, key2 in zip_longest(keys1, keys2): - if key1 is None: - log.critical('key=%s is missing from zone=%s', key1.name, - zone1.name) - assert False - if key2 is None: - log.critical('key=%s is missing from zone=%s', key2.name, - zone2.name) - assert False + # upload more objects + for objname in ('i', 'j', 'k', 'l'): + k = new_key(zone, test_bucket.name, objname) + k.set_contents_from_string('foo') + zonegroup_bucket_checkpoint(zonegroup_conns, test_bucket.name) + + # verify the bucket has non-empty bilog + test_bilog = bilog_list(zone.zone, test_bucket.name) + assert(len(test_bilog) > 0) @attr('bucket_reshard') def test_bucket_sync_run_basic_incremental(): @@ -2269,38 +2509,6 @@ def remove_sync_group_pipe(cluster, group, pipe_id, bucket = None, args = None): assert False, 'failed to remove sync group pipe groupid=%s, pipe_id=%s, src_zones=%s, dest_zones=%s, bucket=%s' % (group, pipe_id, src_zones, dest_zones, bucket) return json.loads(result_json) -def create_zone_bucket(zone): - b_name = gen_bucket_name() - log.info('create bucket zone=%s name=%s', zone.name, b_name) - bucket = zone.create_bucket(b_name) - return bucket - -def create_object(zone_conn, bucket, objname, content): - k = new_key(zone_conn, bucket.name, objname) - k.set_contents_from_string(content) - -def create_objects(zone_conn, bucket, obj_arr, content): - for objname in obj_arr: - create_object(zone_conn, bucket, objname, content) - -def check_object_exists(bucket, objname, content = None): - k = bucket.get_key(objname) - assert_not_equal(k, None) - if (content != None): - assert_equal(k.get_contents_as_string(encoding='ascii'), content) - -def check_objects_exist(bucket, obj_arr, content = None): - for objname in obj_arr: - check_object_exists(bucket, objname, content) - -def check_object_not_exists(bucket, objname): - k = bucket.get_key(objname) - assert_equal(k, None) - -def check_objects_not_exist(bucket, obj_arr): - for objname in obj_arr: - check_object_not_exists(bucket, objname) - @attr('sync_policy') def test_sync_policy_config_zonegroup(): """ @@ -3297,173 +3505,5 @@ def test_sync_flow_symmetrical_zonegroup_all_rgw_down(): test_sync_flow_symmetrical_zonegroup_all() finally: start_2nd_rgw(zonegroup) +''' -def test_topic_notification_sync(): - zonegroup = realm.master_zonegroup() - zonegroup_meta_checkpoint(zonegroup) - # let wait for users and other settings to sync across all zones. - time.sleep(config.checkpoint_delay) - # create topics in each zone. - zonegroup_conns = ZonegroupConns(zonegroup) - topic_arns, zone_topic = create_topic_per_zone(zonegroup_conns) - log.debug("topic_arns: %s", topic_arns) - - zonegroup_meta_checkpoint(zonegroup) - - # verify topics exists in all zones - for conn in zonegroup_conns.zones: - topic_list = conn.list_topics() - log.debug("topics for zone=%s = %s", conn.name, topic_list) - assert_equal(len(topic_list), len(topic_arns)) - for topic_arn_map in topic_list: - assert_true(topic_arn_map['TopicArn'] in topic_arns) - - # create a bucket - bucket = zonegroup_conns.rw_zones[0].create_bucket(gen_bucket_name()) - log.debug('created bucket=%s', bucket.name) - zonegroup_meta_checkpoint(zonegroup) - - # create bucket_notification in each zone. - notification_ids = [] - num = 1 - for zone_conn, topic_arn in zone_topic: - noti_id = "bn" + '-' + run_prefix + '-' + str(num) - notification_ids.append(noti_id) - topic_conf = {'Id': noti_id, - 'TopicArn': topic_arn, - 'Events': ['s3:ObjectCreated:*'] - } - num += 1 - log.info('creating bucket notification for zone=%s name=%s', zone_conn.name, noti_id) - zone_conn.create_notification(bucket.name, [topic_conf]) - zonegroup_meta_checkpoint(zonegroup) - - # verify notifications exists in all zones - for conn in zonegroup_conns.zones: - notification_list = conn.list_notifications(bucket.name) - log.debug("notifications for zone=%s = %s", conn.name, notification_list) - assert_equal(len(notification_list), len(topic_arns)) - for notification in notification_list: - assert_true(notification['Id'] in notification_ids) - - # verify bucket_topic mapping - # create a new bucket and subcribe it to first topic. - bucket_2 = zonegroup_conns.rw_zones[0].create_bucket(gen_bucket_name()) - notif_id = "bn-2" + '-' + run_prefix - topic_conf = {'Id': notif_id, - 'TopicArn': topic_arns[0], - 'Events': ['s3:ObjectCreated:*'] - } - zonegroup_conns.rw_zones[0].create_notification(bucket_2.name, [topic_conf]) - zonegroup_meta_checkpoint(zonegroup) - for conn in zonegroup_conns.zones: - topics = get_topics(conn.zone) - for topic in topics: - if topic['arn'] == topic_arns[0]: - assert_equal(len(topic['subscribed_buckets']), 2) - assert_true(bucket_2.name in topic['subscribed_buckets']) - else: - assert_equal(len(topic['subscribed_buckets']), 1) - assert_true(bucket.name in topic['subscribed_buckets']) - - # delete the 2nd bucket and verify the mapping is removed. - zonegroup_conns.rw_zones[0].delete_bucket(bucket_2.name) - zonegroup_meta_checkpoint(zonegroup) - for conn in zonegroup_conns.zones: - topics = get_topics(conn.zone) - for topic in topics: - assert_equal(len(topic['subscribed_buckets']), 1) - '''TODO(Remove the break once the https://tracker.ceph.com/issues/20802 - is fixed, as the secondary site bucket instance info is currently not - getting deleted coz of the bug hence the bucket-topic mapping - deletion is not invoked on secondary sites.)''' - break - - # delete notifications - zonegroup_conns.rw_zones[0].delete_notifications(bucket.name) - log.debug('Deleting all notifications for bucket=%s', bucket.name) - zonegroup_meta_checkpoint(zonegroup) - - # verify notification deleted in all zones - for conn in zonegroup_conns.zones: - notification_list = conn.list_notifications(bucket.name) - assert_equal(len(notification_list), 0) - - # delete topics - for zone_conn, topic_arn in zone_topic: - log.debug('deleting topic zone=%s arn=%s', zone_conn.name, topic_arn) - zone_conn.delete_topic(topic_arn) - zonegroup_meta_checkpoint(zonegroup) - - # verify topics deleted in all zones - for conn in zonegroup_conns.zones: - topic_list = conn.list_topics() - assert_equal(len(topic_list), 0) - -def test_account_metadata_sync(): - zonegroup = realm.master_zonegroup() - zonegroup_conns = ZonegroupConns(zonegroup) - - inline_policy = json.dumps({'Version': '2012-10-17', 'Statement': [{'Effect': 'Allow', 'Action': 's3:*', 'Resource': '*'}]}) - managed_policy_arn = 'arn:aws:iam::aws:policy/AmazonS3FullAccess' - - for source_conn in zonegroup_conns.rw_zones: - iam = source_conn.iam_conn - name = source_conn.name - # create user, add access key, user policy, managed policy - iam.create_user(UserName=name) - iam.create_access_key(UserName=name) - iam.put_user_policy(UserName=name, PolicyName='Allow', PolicyDocument=inline_policy) - iam.attach_user_policy(UserName=name, PolicyArn=managed_policy_arn) - # create group, group policy, managed policy, add user to group - iam.create_group(GroupName=name) - iam.put_group_policy(GroupName=name, PolicyName='Allow', PolicyDocument=inline_policy) - iam.attach_group_policy(GroupName=name, PolicyArn=managed_policy_arn) - iam.add_user_to_group(GroupName=name, UserName=name) - # create role, role policy, managed policy - iam.create_role(RoleName=name, AssumeRolePolicyDocument=json.dumps({'Version': '2012-10-17', 'Statement': [{'Effect': 'Allow', 'Principal': {'AWS': 'arn:aws:iam:::user/testuser'}, 'Action': ['sts:AssumeRole']}]})) - iam.put_role_policy(RoleName=name, PolicyName='Allow', PolicyDocument=inline_policy) - iam.attach_role_policy(RoleName=name, PolicyArn=managed_policy_arn) - # TODO: test oidc provider - #iam.create_open_id_connect_provider(ClientIDList=['clientid'], ThumbprintList=['3768084dfb3d2b68b7897bf5f565da8efEXAMPLE'], Url=f'http://{name}.example.com') - - realm_meta_checkpoint(realm) - - # check that all users/groups/roles are equal across all zones - for source_conn, target_conn in combinations(zonegroup_conns.zones, 2): - if target_conn.zone.has_roles(): - check_roles_eq(source_conn, target_conn) - check_users_eq(source_conn, target_conn) - check_groups_eq(source_conn, target_conn) - check_oidc_providers_eq(source_conn, target_conn) - - for source_conn in zonegroup_conns.rw_zones: - iam = source_conn.iam_conn - name = source_conn.name - - #iam.delete_open_id_connect_provider(OpenIDConnectProviderArn=f'arn:aws:iam::RGW11111111111111111:oidc-provider/{name}.example.com') - - iam.detach_role_policy(RoleName=name, PolicyArn=managed_policy_arn) - iam.delete_role_policy(RoleName=name, PolicyName='Allow') - iam.delete_role(RoleName=name) - - iam.remove_user_from_group(GroupName=name, UserName=name) - iam.detach_group_policy(GroupName=name, PolicyArn=managed_policy_arn) - iam.delete_group_policy(GroupName=name, PolicyName='Allow') - iam.delete_group(GroupName=name) - - iam.detach_user_policy(UserName=name, PolicyArn=managed_policy_arn) - iam.delete_user_policy(UserName=name, PolicyName='Allow') - key_id = iam.list_access_keys(UserName=name)['AccessKeyMetadata'][0]['AccessKeyId'] - iam.delete_access_key(UserName=name, AccessKeyId=key_id) - iam.delete_user(UserName=name) - - realm_meta_checkpoint(realm) - - # check that all users/groups/roles are equal across all zones - for source_conn, target_conn in combinations(zonegroup_conns.zones, 2): - if target_conn.zone.has_roles(): - check_roles_eq(source_conn, target_conn) - check_users_eq(source_conn, target_conn) - check_groups_eq(source_conn, target_conn) - check_oidc_providers_eq(source_conn, target_conn) From ca460de93f42398e2631ab504b141e11373af1fb Mon Sep 17 00:00:00 2001 From: Casey Bodley Date: Mon, 15 Apr 2024 10:06:11 -0400 Subject: [PATCH 6/9] qa/rgw/multisite: enable notification_v2 feature Signed-off-by: Casey Bodley (cherry picked from commit 1e7fc66683e4dea7294f0ec95f47b22f48b33b7e) --- qa/suites/rgw/multisite/realms/three-zones.yaml | 2 +- qa/suites/rgw/multisite/realms/two-zonegroup.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/qa/suites/rgw/multisite/realms/three-zones.yaml b/qa/suites/rgw/multisite/realms/three-zones.yaml index 95318b0f8a691..06f4cb489098e 100644 --- a/qa/suites/rgw/multisite/realms/three-zones.yaml +++ b/qa/suites/rgw/multisite/realms/three-zones.yaml @@ -8,7 +8,7 @@ overrides: is_master: true is_default: true endpoints: [c1.client.0] - enabled_features: ['resharding'] + enabled_features: ['resharding', 'notification_v2'] zones: - name: test-zone1 is_master: true diff --git a/qa/suites/rgw/multisite/realms/two-zonegroup.yaml b/qa/suites/rgw/multisite/realms/two-zonegroup.yaml index 02710a7d9dd66..0836a953d74f2 100644 --- a/qa/suites/rgw/multisite/realms/two-zonegroup.yaml +++ b/qa/suites/rgw/multisite/realms/two-zonegroup.yaml @@ -8,7 +8,7 @@ overrides: is_master: true is_default: true endpoints: [c1.client.0] - enabled_features: ['resharding'] + enabled_features: ['resharding', 'notification_v2'] zones: - name: a1 is_master: true @@ -19,7 +19,7 @@ overrides: - name: b is_default: true endpoints: [c2.client.0] - enabled_features: ['resharding'] + enabled_features: ['resharding', 'notification_v2'] zones: - name: b1 is_master: true From fb3186266ee52b352ebdf4ce394d3144a23db8f2 Mon Sep 17 00:00:00 2001 From: Shilpa Jagannath Date: Mon, 15 Apr 2024 10:54:57 -0400 Subject: [PATCH 7/9] qa/multisite: disable two-zonegroup and three-zone configs. Signed-off-by: Shilpa Jagannath --- ...e-zones.yaml => three-zones.yaml.disabled} | 0 ...group.yaml => two-zonegroup.yaml.disabled} | 0 qa/suites/rgw/multisite/realms/two-zones.yaml | 4 +- src/test/rgw/rgw_multi/tests.py | 37 +++++++++++++++++-- 4 files changed, 35 insertions(+), 6 deletions(-) rename qa/suites/rgw/multisite/realms/{three-zones.yaml => three-zones.yaml.disabled} (100%) rename qa/suites/rgw/multisite/realms/{two-zonegroup.yaml => two-zonegroup.yaml.disabled} (100%) diff --git a/qa/suites/rgw/multisite/realms/three-zones.yaml b/qa/suites/rgw/multisite/realms/three-zones.yaml.disabled similarity index 100% rename from qa/suites/rgw/multisite/realms/three-zones.yaml rename to qa/suites/rgw/multisite/realms/three-zones.yaml.disabled diff --git a/qa/suites/rgw/multisite/realms/two-zonegroup.yaml b/qa/suites/rgw/multisite/realms/two-zonegroup.yaml.disabled similarity index 100% rename from qa/suites/rgw/multisite/realms/two-zonegroup.yaml rename to qa/suites/rgw/multisite/realms/two-zonegroup.yaml.disabled diff --git a/qa/suites/rgw/multisite/realms/two-zones.yaml b/qa/suites/rgw/multisite/realms/two-zones.yaml index 3564c9619b7cb..1bea381077c00 100644 --- a/qa/suites/rgw/multisite/realms/two-zones.yaml +++ b/qa/suites/rgw/multisite/realms/two-zones.yaml @@ -8,7 +8,7 @@ overrides: is_master: true is_default: true endpoints: [c1.client.0] - enabled_features: ['resharding'] + enabled_features: ['resharding', 'notification_v2'] zones: - name: test-zone1 is_master: true @@ -18,4 +18,4 @@ overrides: is_default: true endpoints: [c2.client.0] rgw-multisite-tests: - args: [tests.py] + args: [tests.py, -a, '!fails_with_rgw'] diff --git a/src/test/rgw/rgw_multi/tests.py b/src/test/rgw/rgw_multi/tests.py index 33c885d239bfb..685e0bcb2febd 100644 --- a/src/test/rgw/rgw_multi/tests.py +++ b/src/test/rgw/rgw_multi/tests.py @@ -1919,9 +1919,10 @@ def test_account_metadata_sync(): check_oidc_providers_eq(source_conn, target_conn) -# TODO: move sync policy and endpoint tests into separate test file -# fix data sync init tests -''' +# TODO: disable failing tests temporarily +# until they are fixed + +@attr('fails_with_rgw') def test_bucket_reshard_index_log_trim(): zonegroup = realm.master_zonegroup() zonegroup_conns = ZonegroupConns(zonegroup) @@ -2006,6 +2007,7 @@ def make_test_bucket(): test_bilog = bilog_list(zone.zone, test_bucket.name) assert(len(test_bilog) > 0) +@attr('fails_with_rgw') @attr('bucket_reshard') def test_bucket_sync_run_basic_incremental(): """ @@ -2071,6 +2073,7 @@ def trash_bucket(zone, bucket_name): cmd += ['--bucket', bucket_name] zone.cluster.admin(cmd) +@attr('fails_with_rgw') @attr('bucket_reshard') def test_zap_init_bucket_sync_run(): """ @@ -2223,6 +2226,7 @@ def test_bucket_full_sync_after_data_sync_init(): zonegroup_bucket_checkpoint(zonegroup_conns, bucket.name) zonegroup_data_checkpoint(zonegroup_conns) +@attr('fails_with_rgw') @attr('data_sync_init') @attr('bucket_reshard') def test_resharded_bucket_full_sync_after_data_sync_init(): @@ -2263,6 +2267,7 @@ def test_resharded_bucket_full_sync_after_data_sync_init(): zonegroup_bucket_checkpoint(zonegroup_conns, bucket.name) zonegroup_data_checkpoint(zonegroup_conns) +@attr('fails_with_rgw') @attr('data_sync_init') def test_bucket_incremental_sync_after_data_sync_init(): zonegroup = realm.master_zonegroup() @@ -2300,6 +2305,7 @@ def test_bucket_incremental_sync_after_data_sync_init(): zonegroup_bucket_checkpoint(zonegroup_conns, bucket.name) zonegroup_data_checkpoint(zonegroup_conns) +@attr('fails_with_rgw') @attr('data_sync_init') @attr('bucket_reshard') def test_resharded_bucket_incremental_sync_latest_after_data_sync_init(): @@ -2348,6 +2354,7 @@ def test_resharded_bucket_incremental_sync_latest_after_data_sync_init(): zonegroup_bucket_checkpoint(zonegroup_conns, bucket.name) zonegroup_data_checkpoint(zonegroup_conns) +@attr('fails_with_rgw') @attr('data_sync_init') @attr('bucket_reshard') def test_resharded_bucket_incremental_sync_oldest_after_data_sync_init(): @@ -2509,6 +2516,7 @@ def remove_sync_group_pipe(cluster, group, pipe_id, bucket = None, args = None): assert False, 'failed to remove sync group pipe groupid=%s, pipe_id=%s, src_zones=%s, dest_zones=%s, bucket=%s' % (group, pipe_id, src_zones, dest_zones, bucket) return json.loads(result_json) +@attr('fails_with_rgw') @attr('sync_policy') def test_sync_policy_config_zonegroup(): """ @@ -2580,6 +2588,7 @@ def test_sync_policy_config_zonegroup(): return +@attr('fails_with_rgw') @attr('sync_policy') def test_sync_flow_symmetrical_zonegroup_all(): """ @@ -2637,6 +2646,7 @@ def test_sync_flow_symmetrical_zonegroup_all(): remove_sync_policy_group(c1, "sync-group") return +@attr('fails_with_rgw') @attr('sync_policy') def test_sync_flow_symmetrical_zonegroup_select(): """ @@ -2705,6 +2715,7 @@ def test_sync_flow_symmetrical_zonegroup_select(): remove_sync_policy_group(c1, "sync-group") return +@attr('fails_with_rgw') @attr('sync_policy') def test_sync_flow_directional_zonegroup_select(): """ @@ -2822,6 +2833,7 @@ def test_sync_flow_directional_zonegroup_select(): remove_sync_policy_group(c1, "sync-group") return +@attr('fails_with_rgw') @attr('sync_policy') def test_sync_single_bucket(): """ @@ -2934,6 +2946,7 @@ def test_sync_single_bucket(): remove_sync_policy_group(c1, "sync-group") return +@attr('fails_with_rgw') @attr('sync_policy') def test_sync_different_buckets(): """ @@ -3083,6 +3096,7 @@ def test_sync_different_buckets(): remove_sync_policy_group(c1, "sync-group") return +@attr('fails_with_rgw') @attr('sync_policy') def test_sync_multiple_buckets_to_single(): """ @@ -3204,6 +3218,7 @@ def test_sync_multiple_buckets_to_single(): remove_sync_policy_group(c1, "sync-group") return +@attr('fails_with_rgw') @attr('sync_policy') def test_sync_single_bucket_to_multiple(): """ @@ -3332,6 +3347,7 @@ def start_2nd_rgw(zonegroup): z.gateways[1].start() log.info('gateway started zone=%s gateway=%s', z.name, z.gateways[1].endpoint()) +@attr('fails_with_rgw') @attr('rgw_down') def test_bucket_create_rgw_down(): zonegroup = realm.master_zonegroup() @@ -3349,6 +3365,7 @@ def test_bucket_create_rgw_down(): finally: start_2nd_rgw(zonegroup) +@attr('fails_with_rgw') @attr('rgw_down') def test_bucket_remove_rgw_down(): zonegroup = realm.master_zonegroup() @@ -3374,6 +3391,7 @@ def test_bucket_remove_rgw_down(): finally: start_2nd_rgw(zonegroup) +@attr('fails_with_rgw') @attr('rgw_down') def test_object_sync_rgw_down(): zonegroup = realm.master_zonegroup() @@ -3385,6 +3403,7 @@ def test_object_sync_rgw_down(): finally: start_2nd_rgw(zonegroup) +@attr('fails_with_rgw') @attr('rgw_down') def test_object_delete_rgw_down(): zonegroup = realm.master_zonegroup() @@ -3396,6 +3415,7 @@ def test_object_delete_rgw_down(): finally: start_2nd_rgw(zonegroup) +@attr('fails_with_rgw') @attr('rgw_down') def test_concurrent_versioned_object_incremental_sync_rgw_down(): zonegroup = realm.master_zonegroup() @@ -3407,6 +3427,7 @@ def test_concurrent_versioned_object_incremental_sync_rgw_down(): finally: start_2nd_rgw(zonegroup) +@attr('fails_with_rgw') @attr('rgw_down') def test_suspended_delete_marker_full_sync_rgw_down(): zonegroup = realm.master_zonegroup() @@ -3418,6 +3439,7 @@ def test_suspended_delete_marker_full_sync_rgw_down(): finally: start_2nd_rgw(zonegroup) +@attr('fails_with_rgw') @attr('rgw_down') def test_bucket_acl_rgw_down(): zonegroup = realm.master_zonegroup() @@ -3429,6 +3451,7 @@ def test_bucket_acl_rgw_down(): finally: start_2nd_rgw(zonegroup) +@attr('fails_with_rgw') @attr('rgw_down') def test_bucket_sync_enable_right_after_disable_rgw_down(): zonegroup = realm.master_zonegroup() @@ -3440,6 +3463,7 @@ def test_bucket_sync_enable_right_after_disable_rgw_down(): finally: start_2nd_rgw(zonegroup) +@attr('fails_with_rgw') @attr('rgw_down') def test_multipart_object_sync_rgw_down(): zonegroup = realm.master_zonegroup() @@ -3451,6 +3475,7 @@ def test_multipart_object_sync_rgw_down(): finally: start_2nd_rgw(zonegroup) +@attr('fails_with_rgw') @attr('rgw_down') def test_bucket_sync_run_basic_incremental_rgw_down(): zonegroup = realm.master_zonegroup() @@ -3462,6 +3487,7 @@ def test_bucket_sync_run_basic_incremental_rgw_down(): finally: start_2nd_rgw(zonegroup) +@attr('fails_with_rgw') @attr('rgw_down') def test_role_sync_rgw_down(): zonegroup = realm.master_zonegroup() @@ -3473,6 +3499,7 @@ def test_role_sync_rgw_down(): finally: start_2nd_rgw(zonegroup) +@attr('fails_with_rgw') @attr('rgw_down') def test_bucket_full_sync_after_data_sync_init_rgw_down(): zonegroup = realm.master_zonegroup() @@ -3484,6 +3511,7 @@ def test_bucket_full_sync_after_data_sync_init_rgw_down(): finally: start_2nd_rgw(zonegroup) +@attr('fails_with_rgw') @attr('rgw_down') def test_sync_policy_config_zonegroup_rgw_down(): zonegroup = realm.master_zonegroup() @@ -3495,6 +3523,7 @@ def test_sync_policy_config_zonegroup_rgw_down(): finally: start_2nd_rgw(zonegroup) +@attr('fails_with_rgw') @attr('rgw_down') def test_sync_flow_symmetrical_zonegroup_all_rgw_down(): zonegroup = realm.master_zonegroup() @@ -3505,5 +3534,5 @@ def test_sync_flow_symmetrical_zonegroup_all_rgw_down(): test_sync_flow_symmetrical_zonegroup_all() finally: start_2nd_rgw(zonegroup) -''' + From da788bc28dbf19bbb54fa91b8c3867063b711551 Mon Sep 17 00:00:00 2001 From: Shilpa Jagannath Date: Tue, 21 May 2024 10:31:31 -0700 Subject: [PATCH 8/9] multisite/qa: disable sync error injection Signed-off-by: Shilpa Jagannath --- qa/suites/rgw/multisite/overrides.yaml | 4 ++-- src/test/rgw/rgw_multi/tests.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/qa/suites/rgw/multisite/overrides.yaml b/qa/suites/rgw/multisite/overrides.yaml index 49f95afb00f49..fad120bf2ed9b 100644 --- a/qa/suites/rgw/multisite/overrides.yaml +++ b/qa/suites/rgw/multisite/overrides.yaml @@ -18,7 +18,7 @@ overrides: rgw data sync poll interval: 5 rgw meta sync poll interval: 5 rgw sync obj etag verify: true - rgw sync meta inject err probability: 0.1 - rgw sync data inject err probability: 0.1 + rgw sync meta inject err probability: 0 + rgw sync data inject err probability: 0 rgw: compression type: random diff --git a/src/test/rgw/rgw_multi/tests.py b/src/test/rgw/rgw_multi/tests.py index 685e0bcb2febd..2b3e207752c07 100644 --- a/src/test/rgw/rgw_multi/tests.py +++ b/src/test/rgw/rgw_multi/tests.py @@ -2195,6 +2195,7 @@ def test_object_acl(): after_set_acl = bucket2.get_acl(k) assert(len(after_set_acl.acl.grants) == 2) # read grant added on AllUsers +@attr('fails_with_rgw') @attr('data_sync_init') def test_bucket_full_sync_after_data_sync_init(): zonegroup = realm.master_zonegroup() From 6753e27daab5135973fa45e5a45a4e8e5fbac101 Mon Sep 17 00:00:00 2001 From: Shilpa Jagannath Date: Thu, 6 Jun 2024 12:06:49 -0700 Subject: [PATCH 9/9] multisite/qa: resolve rebase conflicts Signed-off-by: Shilpa Jagannath --- src/test/rgw/rgw_multi/tests.py | 704 +++++++++++++++----------------- 1 file changed, 334 insertions(+), 370 deletions(-) diff --git a/src/test/rgw/rgw_multi/tests.py b/src/test/rgw/rgw_multi/tests.py index 2b3e207752c07..df9c70a47810a 100644 --- a/src/test/rgw/rgw_multi/tests.py +++ b/src/test/rgw/rgw_multi/tests.py @@ -765,38 +765,6 @@ def check_oidc_providers_eq(zone_conn1, zone_conn2): eq(p1, p2) check_oidc_provider_eq(zone_conn1, zone_conn2, p1['Arn']) -def create_zone_bucket(zone): - b_name = gen_bucket_name() - log.info('create bucket zone=%s name=%s', zone.name, b_name) - bucket = zone.create_bucket(b_name) - return bucket - -def create_object(zone_conn, bucket, objname, content): - k = new_key(zone_conn, bucket.name, objname) - k.set_contents_from_string(content) - -def create_objects(zone_conn, bucket, obj_arr, content): - for objname in obj_arr: - create_object(zone_conn, bucket, objname, content) - -def check_object_exists(bucket, objname, content = None): - k = bucket.get_key(objname) - assert_not_equal(k, None) - if (content != None): - assert_equal(k.get_contents_as_string(encoding='ascii'), content) - -def check_objects_exist(bucket, obj_arr, content = None): - for objname in obj_arr: - check_object_exists(bucket, objname, content) - -def check_object_not_exists(bucket, objname): - k = bucket.get_key(objname) - assert_equal(k, None) - -def check_objects_not_exist(bucket, obj_arr): - for objname in obj_arr: - check_object_not_exists(bucket, objname) - def test_object_sync(): zonegroup = realm.master_zonegroup() zonegroup_conns = ZonegroupConns(zonegroup) @@ -1568,6 +1536,97 @@ def make_test_bucket(): cold_bilog = bilog_list(zone.zone, cold_bucket.name) assert(len(cold_bilog) == 0) +# TODO: disable failing tests temporarily +# until they are fixed + +@attr('fails_with_rgw') +def test_bucket_reshard_index_log_trim(): + zonegroup = realm.master_zonegroup() + zonegroup_conns = ZonegroupConns(zonegroup) + + zone = zonegroup_conns.rw_zones[0] + + # create a test bucket, upload some objects, and wait for sync + def make_test_bucket(): + name = gen_bucket_name() + log.info('create bucket zone=%s name=%s', zone.name, name) + bucket = zone.conn.create_bucket(name) + for objname in ('a', 'b', 'c', 'd'): + k = new_key(zone, name, objname) + k.set_contents_from_string('foo') + zonegroup_meta_checkpoint(zonegroup) + zonegroup_bucket_checkpoint(zonegroup_conns, name) + return bucket + + # create a 'test' bucket + test_bucket = make_test_bucket() + + # checking bucket layout before resharding + json_obj_1 = bucket_layout(zone.zone, test_bucket.name) + assert(len(json_obj_1['layout']['logs']) == 1) + + first_gen = json_obj_1['layout']['current_index']['gen'] + + before_reshard_bilog = bilog_list(zone.zone, test_bucket.name, ['--gen', str(first_gen)]) + assert(len(before_reshard_bilog) == 4) + + # Resharding the bucket + zone.zone.cluster.admin(['bucket', 'reshard', + '--bucket', test_bucket.name, + '--num-shards', '3', + '--yes-i-really-mean-it']) + + # checking bucket layout after 1st resharding + json_obj_2 = bucket_layout(zone.zone, test_bucket.name) + assert(len(json_obj_2['layout']['logs']) == 2) + + second_gen = json_obj_2['layout']['current_index']['gen'] + + after_reshard_bilog = bilog_list(zone.zone, test_bucket.name, ['--gen', str(second_gen)]) + assert(len(after_reshard_bilog) == 0) + + # upload more objects + for objname in ('e', 'f', 'g', 'h'): + k = new_key(zone, test_bucket.name, objname) + k.set_contents_from_string('foo') + zonegroup_bucket_checkpoint(zonegroup_conns, test_bucket.name) + + # Resharding the bucket again + zone.zone.cluster.admin(['bucket', 'reshard', + '--bucket', test_bucket.name, + '--num-shards', '3', + '--yes-i-really-mean-it']) + + # checking bucket layout after 2nd resharding + json_obj_3 = bucket_layout(zone.zone, test_bucket.name) + assert(len(json_obj_3['layout']['logs']) == 3) + + zonegroup_bucket_checkpoint(zonegroup_conns, test_bucket.name) + + bilog_autotrim(zone.zone) + + # checking bucket layout after 1st bilog autotrim + json_obj_4 = bucket_layout(zone.zone, test_bucket.name) + assert(len(json_obj_4['layout']['logs']) == 2) + + bilog_autotrim(zone.zone) + + # checking bucket layout after 2nd bilog autotrim + json_obj_5 = bucket_layout(zone.zone, test_bucket.name) + assert(len(json_obj_5['layout']['logs']) == 1) + + bilog_autotrim(zone.zone) + + # upload more objects + for objname in ('i', 'j', 'k', 'l'): + k = new_key(zone, test_bucket.name, objname) + k.set_contents_from_string('foo') + zonegroup_bucket_checkpoint(zonegroup_conns, test_bucket.name) + + # verify the bucket has non-empty bilog + test_bilog = bilog_list(zone.zone, test_bucket.name) + assert(len(test_bilog) > 0) + @attr('bucket_reshard') def test_bucket_reshard_incremental(): zonegroup = realm.master_zonegroup() @@ -1664,348 +1723,53 @@ def write_most_shards(zone, bucket_name, num_shards): Write one object to most (but not all) bucket index shards. """ objs = get_bucket_shard_objects(zone.zone, num_shards) - random.shuffle(objs) - del objs[-(len(objs)//10):] - for obj in objs: - k = new_key(zone, bucket_name, obj) - k.set_contents_from_string('foo') - -def reshard_bucket(zone, bucket_name, num_shards): - """ - Reshard a bucket - """ - cmd = ['bucket', 'reshard'] + zone.zone_args() - cmd += ['--bucket', bucket_name] - cmd += ['--num-shards', str(num_shards)] - cmd += ['--yes-i-really-mean-it'] - zone.cluster.admin(cmd) - -def get_obj_names(zone, bucket_name, maxobjs): - """ - Get names of objects in a bucket. - """ - cmd = ['bucket', 'list'] + zone.zone_args() - cmd += ['--bucket', bucket_name] - cmd += ['--max-entries', str(maxobjs)] - objs_json, _ = zone.cluster.admin(cmd, read_only=True) - objs = json.loads(objs_json) - return [o['name'] for o in objs] - -def bucket_keys_eq(zone1, zone2, bucket_name): - """ - Ensure that two buckets have the same keys, but get the lists through - radosgw-admin rather than S3 so it can be used when radosgw isn't running. - Only works for buckets of 10,000 objects since the tests calling it don't - need more, and the output from bucket list doesn't have an obvious marker - with which to continue. - """ - keys1 = get_obj_names(zone1, bucket_name, 10000) - keys2 = get_obj_names(zone2, bucket_name, 10000) - for key1, key2 in zip_longest(keys1, keys2): - if key1 is None: - log.critical('key=%s is missing from zone=%s', key1.name, - zone1.name) - assert False - if key2 is None: - log.critical('key=%s is missing from zone=%s', key2.name, - zone2.name) - assert False - -def test_role_sync(): - zonegroup = realm.master_zonegroup() - zonegroup_conns = ZonegroupConns(zonegroup) - roles, zone_role = create_role_per_zone(zonegroup_conns) - - zonegroup_meta_checkpoint(zonegroup) - - for source_conn, target_conn in combinations(zonegroup_conns.zones, 2): - if target_conn.zone.has_roles(): - check_roles_eq(source_conn, target_conn) - -def test_role_delete_sync(): - zonegroup = realm.master_zonegroup() - zonegroup_conns = ZonegroupConns(zonegroup) - role_name = gen_role_name() - log.info('create role zone=%s name=%s', zonegroup_conns.master_zone.name, role_name) - policy_document = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"arn:aws:iam:::user/testuser\"]},\"Action\":[\"sts:AssumeRole\"]}]}" - zonegroup_conns.master_zone.iam_conn.create_role(RoleName=role_name, AssumeRolePolicyDocument=policy_document) - - zonegroup_meta_checkpoint(zonegroup) - - for zone in zonegroup_conns.zones: - log.info(f'checking if zone: {zone.name} has role: {role_name}') - zone.iam_conn.get_role(RoleName=role_name) - log.info(f'success, zone: {zone.name} has role: {role_name}') - - log.info(f"deleting role: {role_name}") - zonegroup_conns.master_zone.iam_conn.delete_role(RoleName=role_name) - zonegroup_meta_checkpoint(zonegroup) - - for zone in zonegroup_conns.zones: - log.info(f'checking if zone: {zone.name} does not have role: {role_name}') - assert_raises(zone.iam_conn.exceptions.NoSuchEntityException, - zone.iam_conn.get_role, RoleName=role_name) - log.info(f'success, zone: {zone.name} does not have role: {role_name}') - -@attr('topic notification') -def test_topic_notification_sync(): - zonegroup = realm.master_zonegroup() - zonegroup_meta_checkpoint(zonegroup) - # let wait for users and other settings to sync across all zones. - time.sleep(config.checkpoint_delay) - # create topics in each zone. - zonegroup_conns = ZonegroupConns(zonegroup) - topic_arns, zone_topic = create_topic_per_zone(zonegroup_conns) - log.debug("topic_arns: %s", topic_arns) - - zonegroup_meta_checkpoint(zonegroup) - - # verify topics exists in all zones - for conn in zonegroup_conns.zones: - topic_list = conn.list_topics() - log.debug("topics for zone=%s = %s", conn.name, topic_list) - assert_equal(len(topic_list), len(topic_arns)) - for topic_arn_map in topic_list: - assert_true(topic_arn_map['TopicArn'] in topic_arns) - - # create a bucket - bucket = zonegroup_conns.rw_zones[0].create_bucket(gen_bucket_name()) - log.debug('created bucket=%s', bucket.name) - zonegroup_meta_checkpoint(zonegroup) - - # create bucket_notification in each zone. - notification_ids = [] - num = 1 - for zone_conn, topic_arn in zone_topic: - noti_id = "bn" + '-' + run_prefix + '-' + str(num) - notification_ids.append(noti_id) - topic_conf = {'Id': noti_id, - 'TopicArn': topic_arn, - 'Events': ['s3:ObjectCreated:*'] - } - num += 1 - log.info('creating bucket notification for zone=%s name=%s', zone_conn.name, noti_id) - zone_conn.create_notification(bucket.name, [topic_conf]) - zonegroup_meta_checkpoint(zonegroup) - - # verify notifications exists in all zones - for conn in zonegroup_conns.zones: - notification_list = conn.list_notifications(bucket.name) - log.debug("notifications for zone=%s = %s", conn.name, notification_list) - assert_equal(len(notification_list), len(topic_arns)) - for notification in notification_list: - assert_true(notification['Id'] in notification_ids) - - # verify bucket_topic mapping - # create a new bucket and subcribe it to first topic. - bucket_2 = zonegroup_conns.rw_zones[0].create_bucket(gen_bucket_name()) - notif_id = "bn-2" + '-' + run_prefix - topic_conf = {'Id': notif_id, - 'TopicArn': topic_arns[0], - 'Events': ['s3:ObjectCreated:*'] - } - zonegroup_conns.rw_zones[0].create_notification(bucket_2.name, [topic_conf]) - zonegroup_meta_checkpoint(zonegroup) - for conn in zonegroup_conns.zones: - topics = get_topics(conn.zone) - for topic in topics: - if topic['arn'] == topic_arns[0]: - assert_equal(len(topic['subscribed_buckets']), 2) - assert_true(bucket_2.name in topic['subscribed_buckets']) - else: - assert_equal(len(topic['subscribed_buckets']), 1) - assert_true(bucket.name in topic['subscribed_buckets']) - - # delete the 2nd bucket and verify the mapping is removed. - zonegroup_conns.rw_zones[0].delete_bucket(bucket_2.name) - zonegroup_meta_checkpoint(zonegroup) - for conn in zonegroup_conns.zones: - topics = get_topics(conn.zone) - for topic in topics: - assert_equal(len(topic['subscribed_buckets']), 1) - '''TODO(Remove the break once the https://tracker.ceph.com/issues/20802 - is fixed, as the secondary site bucket instance info is currently not - getting deleted coz of the bug hence the bucket-topic mapping - deletion is not invoked on secondary sites.)''' - break - - # delete notifications - zonegroup_conns.rw_zones[0].delete_notifications(bucket.name) - log.debug('Deleting all notifications for bucket=%s', bucket.name) - zonegroup_meta_checkpoint(zonegroup) - - # verify notification deleted in all zones - for conn in zonegroup_conns.zones: - notification_list = conn.list_notifications(bucket.name) - assert_equal(len(notification_list), 0) - - # delete topics - for zone_conn, topic_arn in zone_topic: - log.debug('deleting topic zone=%s arn=%s', zone_conn.name, topic_arn) - zone_conn.delete_topic(topic_arn) - zonegroup_meta_checkpoint(zonegroup) - - # verify topics deleted in all zones - for conn in zonegroup_conns.zones: - topic_list = conn.list_topics() - assert_equal(len(topic_list), 0) - -def test_account_metadata_sync(): - zonegroup = realm.master_zonegroup() - zonegroup_conns = ZonegroupConns(zonegroup) - - inline_policy = json.dumps({'Version': '2012-10-17', 'Statement': [{'Effect': 'Allow', 'Action': 's3:*', 'Resource': '*'}]}) - managed_policy_arn = 'arn:aws:iam::aws:policy/AmazonS3FullAccess' - - for source_conn in zonegroup_conns.rw_zones: - iam = source_conn.iam_conn - name = source_conn.name - # create user, add access key, user policy, managed policy - iam.create_user(UserName=name) - iam.create_access_key(UserName=name) - iam.put_user_policy(UserName=name, PolicyName='Allow', PolicyDocument=inline_policy) - iam.attach_user_policy(UserName=name, PolicyArn=managed_policy_arn) - # create group, group policy, managed policy, add user to group - iam.create_group(GroupName=name) - iam.put_group_policy(GroupName=name, PolicyName='Allow', PolicyDocument=inline_policy) - iam.attach_group_policy(GroupName=name, PolicyArn=managed_policy_arn) - iam.add_user_to_group(GroupName=name, UserName=name) - # create role, role policy, managed policy - iam.create_role(RoleName=name, AssumeRolePolicyDocument=json.dumps({'Version': '2012-10-17', 'Statement': [{'Effect': 'Allow', 'Principal': {'AWS': 'arn:aws:iam:::user/testuser'}, 'Action': ['sts:AssumeRole']}]})) - iam.put_role_policy(RoleName=name, PolicyName='Allow', PolicyDocument=inline_policy) - iam.attach_role_policy(RoleName=name, PolicyArn=managed_policy_arn) - # TODO: test oidc provider - #iam.create_open_id_connect_provider(ClientIDList=['clientid'], ThumbprintList=['3768084dfb3d2b68b7897bf5f565da8efEXAMPLE'], Url=f'http://{name}.example.com') - - realm_meta_checkpoint(realm) - - # check that all users/groups/roles are equal across all zones - for source_conn, target_conn in combinations(zonegroup_conns.zones, 2): - if target_conn.zone.has_roles(): - check_roles_eq(source_conn, target_conn) - check_users_eq(source_conn, target_conn) - check_groups_eq(source_conn, target_conn) - check_oidc_providers_eq(source_conn, target_conn) - - for source_conn in zonegroup_conns.rw_zones: - iam = source_conn.iam_conn - name = source_conn.name - - #iam.delete_open_id_connect_provider(OpenIDConnectProviderArn=f'arn:aws:iam::RGW11111111111111111:oidc-provider/{name}.example.com') - - iam.detach_role_policy(RoleName=name, PolicyArn=managed_policy_arn) - iam.delete_role_policy(RoleName=name, PolicyName='Allow') - iam.delete_role(RoleName=name) - - iam.remove_user_from_group(GroupName=name, UserName=name) - iam.detach_group_policy(GroupName=name, PolicyArn=managed_policy_arn) - iam.delete_group_policy(GroupName=name, PolicyName='Allow') - iam.delete_group(GroupName=name) - - iam.detach_user_policy(UserName=name, PolicyArn=managed_policy_arn) - iam.delete_user_policy(UserName=name, PolicyName='Allow') - key_id = iam.list_access_keys(UserName=name)['AccessKeyMetadata'][0]['AccessKeyId'] - iam.delete_access_key(UserName=name, AccessKeyId=key_id) - iam.delete_user(UserName=name) - - realm_meta_checkpoint(realm) - - # check that all users/groups/roles are equal across all zones - for source_conn, target_conn in combinations(zonegroup_conns.zones, 2): - if target_conn.zone.has_roles(): - check_roles_eq(source_conn, target_conn) - check_users_eq(source_conn, target_conn) - check_groups_eq(source_conn, target_conn) - check_oidc_providers_eq(source_conn, target_conn) - - -# TODO: disable failing tests temporarily -# until they are fixed - -@attr('fails_with_rgw') -def test_bucket_reshard_index_log_trim(): - zonegroup = realm.master_zonegroup() - zonegroup_conns = ZonegroupConns(zonegroup) - - zone = zonegroup_conns.rw_zones[0] - - # create a test bucket, upload some objects, and wait for sync - def make_test_bucket(): - name = gen_bucket_name() - log.info('create bucket zone=%s name=%s', zone.name, name) - bucket = zone.conn.create_bucket(name) - for objname in ('a', 'b', 'c', 'd'): - k = new_key(zone, name, objname) - k.set_contents_from_string('foo') - zonegroup_meta_checkpoint(zonegroup) - zonegroup_bucket_checkpoint(zonegroup_conns, name) - return bucket - - # create a 'test' bucket - test_bucket = make_test_bucket() - - # checking bucket layout before resharding - json_obj_1 = bucket_layout(zone.zone, test_bucket.name) - assert(len(json_obj_1['layout']['logs']) == 1) - - first_gen = json_obj_1['layout']['current_index']['gen'] - - before_reshard_bilog = bilog_list(zone.zone, test_bucket.name, ['--gen', str(first_gen)]) - assert(len(before_reshard_bilog) == 4) - - # Resharding the bucket - zone.zone.cluster.admin(['bucket', 'reshard', - '--bucket', test_bucket.name, - '--num-shards', '13']) - - # checking bucket layout after 1st resharding - json_obj_2 = bucket_layout(zone.zone, test_bucket.name) - assert(len(json_obj_2['layout']['logs']) == 2) - - second_gen = json_obj_2['layout']['current_index']['gen'] - - after_reshard_bilog = bilog_list(zone.zone, test_bucket.name, ['--gen', str(second_gen)]) - assert(len(after_reshard_bilog) == 0) - - # upload more objects - for objname in ('e', 'f', 'g', 'h'): - k = new_key(zone, test_bucket.name, objname) - k.set_contents_from_string('foo') - zonegroup_bucket_checkpoint(zonegroup_conns, test_bucket.name) - - # Resharding the bucket again - zone.zone.cluster.admin(['bucket', 'reshard', - '--bucket', test_bucket.name, - '--num-shards', '15']) - - # checking bucket layout after 2nd resharding - json_obj_3 = bucket_layout(zone.zone, test_bucket.name) - assert(len(json_obj_3['layout']['logs']) == 3) - - zonegroup_bucket_checkpoint(zonegroup_conns, test_bucket.name) - - bilog_autotrim(zone.zone) - - # checking bucket layout after 1st bilog autotrim - json_obj_4 = bucket_layout(zone.zone, test_bucket.name) - assert(len(json_obj_4['layout']['logs']) == 2) + random.shuffle(objs) + del objs[-(len(objs)//10):] + for obj in objs: + k = new_key(zone, bucket_name, obj) + k.set_contents_from_string('foo') - bilog_autotrim(zone.zone) +def reshard_bucket(zone, bucket_name, num_shards): + """ + Reshard a bucket + """ + cmd = ['bucket', 'reshard'] + zone.zone_args() + cmd += ['--bucket', bucket_name] + cmd += ['--num-shards', str(num_shards)] + cmd += ['--yes-i-really-mean-it'] + zone.cluster.admin(cmd) - # checking bucket layout after 2nd bilog autotrim - json_obj_5 = bucket_layout(zone.zone, test_bucket.name) - time.sleep(config.checkpoint_delay) - assert(len(json_obj_5['layout']['logs']) == 1) +def get_obj_names(zone, bucket_name, maxobjs): + """ + Get names of objects in a bucket. + """ + cmd = ['bucket', 'list'] + zone.zone_args() + cmd += ['--bucket', bucket_name] + cmd += ['--max-entries', str(maxobjs)] + objs_json, _ = zone.cluster.admin(cmd, read_only=True) + objs = json.loads(objs_json) + return [o['name'] for o in objs] - # upload more objects - for objname in ('i', 'j', 'k', 'l'): - k = new_key(zone, test_bucket.name, objname) - k.set_contents_from_string('foo') - zonegroup_bucket_checkpoint(zonegroup_conns, test_bucket.name) +def bucket_keys_eq(zone1, zone2, bucket_name): + """ + Ensure that two buckets have the same keys, but get the lists through + radosgw-admin rather than S3 so it can be used when radosgw isn't running. + Only works for buckets of 10,000 objects since the tests calling it don't + need more, and the output from bucket list doesn't have an obvious marker + with which to continue. + """ + keys1 = get_obj_names(zone1, bucket_name, 10000) + keys2 = get_obj_names(zone2, bucket_name, 10000) + for key1, key2 in zip_longest(keys1, keys2): + if key1 is None: + log.critical('key=%s is missing from zone=%s', key1.name, + zone1.name) + assert False + if key2 is None: + log.critical('key=%s is missing from zone=%s', key2.name, + zone2.name) + assert False - # verify the bucket has non-empty bilog - test_bilog = bilog_list(zone.zone, test_bucket.name) - assert(len(test_bilog) > 0) @attr('fails_with_rgw') @attr('bucket_reshard') @@ -2517,6 +2281,38 @@ def remove_sync_group_pipe(cluster, group, pipe_id, bucket = None, args = None): assert False, 'failed to remove sync group pipe groupid=%s, pipe_id=%s, src_zones=%s, dest_zones=%s, bucket=%s' % (group, pipe_id, src_zones, dest_zones, bucket) return json.loads(result_json) +def create_zone_bucket(zone): + b_name = gen_bucket_name() + log.info('create bucket zone=%s name=%s', zone.name, b_name) + bucket = zone.create_bucket(b_name) + return bucket + +def create_object(zone_conn, bucket, objname, content): + k = new_key(zone_conn, bucket.name, objname) + k.set_contents_from_string(content) + +def create_objects(zone_conn, bucket, obj_arr, content): + for objname in obj_arr: + create_object(zone_conn, bucket, objname, content) + +def check_object_exists(bucket, objname, content = None): + k = bucket.get_key(objname) + assert_not_equal(k, None) + if (content != None): + assert_equal(k.get_contents_as_string(encoding='ascii'), content) + +def check_objects_exist(bucket, obj_arr, content = None): + for objname in obj_arr: + check_object_exists(bucket, objname, content) + +def check_object_not_exists(bucket, objname): + k = bucket.get_key(objname) + assert_equal(k, None) + +def check_objects_not_exist(bucket, obj_arr): + for objname in obj_arr: + check_object_not_exists(bucket, objname) + @attr('fails_with_rgw') @attr('sync_policy') def test_sync_policy_config_zonegroup(): @@ -3536,4 +3332,172 @@ def test_sync_flow_symmetrical_zonegroup_all_rgw_down(): finally: start_2nd_rgw(zonegroup) +def test_topic_notification_sync(): + zonegroup = realm.master_zonegroup() + zonegroup_meta_checkpoint(zonegroup) + # let wait for users and other settings to sync across all zones. + time.sleep(config.checkpoint_delay) + # create topics in each zone. + zonegroup_conns = ZonegroupConns(zonegroup) + topic_arns, zone_topic = create_topic_per_zone(zonegroup_conns) + log.debug("topic_arns: %s", topic_arns) + + zonegroup_meta_checkpoint(zonegroup) + + # verify topics exists in all zones + for conn in zonegroup_conns.zones: + topic_list = conn.list_topics() + log.debug("topics for zone=%s = %s", conn.name, topic_list) + assert_equal(len(topic_list), len(topic_arns)) + for topic_arn_map in topic_list: + assert_true(topic_arn_map['TopicArn'] in topic_arns) + + # create a bucket + bucket = zonegroup_conns.rw_zones[0].create_bucket(gen_bucket_name()) + log.debug('created bucket=%s', bucket.name) + zonegroup_meta_checkpoint(zonegroup) + + # create bucket_notification in each zone. + notification_ids = [] + num = 1 + for zone_conn, topic_arn in zone_topic: + noti_id = "bn" + '-' + run_prefix + '-' + str(num) + notification_ids.append(noti_id) + topic_conf = {'Id': noti_id, + 'TopicArn': topic_arn, + 'Events': ['s3:ObjectCreated:*'] + } + num += 1 + log.info('creating bucket notification for zone=%s name=%s', zone_conn.name, noti_id) + zone_conn.create_notification(bucket.name, [topic_conf]) + zonegroup_meta_checkpoint(zonegroup) + + # verify notifications exists in all zones + for conn in zonegroup_conns.zones: + notification_list = conn.list_notifications(bucket.name) + log.debug("notifications for zone=%s = %s", conn.name, notification_list) + assert_equal(len(notification_list), len(topic_arns)) + for notification in notification_list: + assert_true(notification['Id'] in notification_ids) + + # verify bucket_topic mapping + # create a new bucket and subcribe it to first topic. + bucket_2 = zonegroup_conns.rw_zones[0].create_bucket(gen_bucket_name()) + notif_id = "bn-2" + '-' + run_prefix + topic_conf = {'Id': notif_id, + 'TopicArn': topic_arns[0], + 'Events': ['s3:ObjectCreated:*'] + } + zonegroup_conns.rw_zones[0].create_notification(bucket_2.name, [topic_conf]) + zonegroup_meta_checkpoint(zonegroup) + for conn in zonegroup_conns.zones: + topics = get_topics(conn.zone) + for topic in topics: + if topic['arn'] == topic_arns[0]: + assert_equal(len(topic['subscribed_buckets']), 2) + assert_true(bucket_2.name in topic['subscribed_buckets']) + else: + assert_equal(len(topic['subscribed_buckets']), 1) + assert_true(bucket.name in topic['subscribed_buckets']) + + # delete the 2nd bucket and verify the mapping is removed. + zonegroup_conns.rw_zones[0].delete_bucket(bucket_2.name) + zonegroup_meta_checkpoint(zonegroup) + for conn in zonegroup_conns.zones: + topics = get_topics(conn.zone) + for topic in topics: + assert_equal(len(topic['subscribed_buckets']), 1) + '''TODO(Remove the break once the https://tracker.ceph.com/issues/20802 + is fixed, as the secondary site bucket instance info is currently not + getting deleted coz of the bug hence the bucket-topic mapping + deletion is not invoked on secondary sites.)''' + break + + # delete notifications + zonegroup_conns.rw_zones[0].delete_notifications(bucket.name) + log.debug('Deleting all notifications for bucket=%s', bucket.name) + zonegroup_meta_checkpoint(zonegroup) + + # verify notification deleted in all zones + for conn in zonegroup_conns.zones: + notification_list = conn.list_notifications(bucket.name) + assert_equal(len(notification_list), 0) + + # delete topics + for zone_conn, topic_arn in zone_topic: + log.debug('deleting topic zone=%s arn=%s', zone_conn.name, topic_arn) + zone_conn.delete_topic(topic_arn) + zonegroup_meta_checkpoint(zonegroup) + + # verify topics deleted in all zones + for conn in zonegroup_conns.zones: + topic_list = conn.list_topics() + assert_equal(len(topic_list), 0) + +def test_account_metadata_sync(): + zonegroup = realm.master_zonegroup() + zonegroup_conns = ZonegroupConns(zonegroup) + + inline_policy = json.dumps({'Version': '2012-10-17', 'Statement': [{'Effect': 'Allow', 'Action': 's3:*', 'Resource': '*'}]}) + managed_policy_arn = 'arn:aws:iam::aws:policy/AmazonS3FullAccess' + + for source_conn in zonegroup_conns.rw_zones: + iam = source_conn.iam_conn + name = source_conn.name + # create user, add access key, user policy, managed policy + iam.create_user(UserName=name) + iam.create_access_key(UserName=name) + iam.put_user_policy(UserName=name, PolicyName='Allow', PolicyDocument=inline_policy) + iam.attach_user_policy(UserName=name, PolicyArn=managed_policy_arn) + # create group, group policy, managed policy, add user to group + iam.create_group(GroupName=name) + iam.put_group_policy(GroupName=name, PolicyName='Allow', PolicyDocument=inline_policy) + iam.attach_group_policy(GroupName=name, PolicyArn=managed_policy_arn) + iam.add_user_to_group(GroupName=name, UserName=name) + # create role, role policy, managed policy + iam.create_role(RoleName=name, AssumeRolePolicyDocument=json.dumps({'Version': '2012-10-17', 'Statement': [{'Effect': 'Allow', 'Principal': {'AWS': 'arn:aws:iam:::user/testuser'}, 'Action': ['sts:AssumeRole']}]})) + iam.put_role_policy(RoleName=name, PolicyName='Allow', PolicyDocument=inline_policy) + iam.attach_role_policy(RoleName=name, PolicyArn=managed_policy_arn) + # TODO: test oidc provider + #iam.create_open_id_connect_provider(ClientIDList=['clientid'], ThumbprintList=['3768084dfb3d2b68b7897bf5f565da8efEXAMPLE'], Url=f'http://{name}.example.com') + + realm_meta_checkpoint(realm) + + # check that all users/groups/roles are equal across all zones + for source_conn, target_conn in combinations(zonegroup_conns.zones, 2): + if target_conn.zone.has_roles(): + check_roles_eq(source_conn, target_conn) + check_users_eq(source_conn, target_conn) + check_groups_eq(source_conn, target_conn) + check_oidc_providers_eq(source_conn, target_conn) + + for source_conn in zonegroup_conns.rw_zones: + iam = source_conn.iam_conn + name = source_conn.name + + #iam.delete_open_id_connect_provider(OpenIDConnectProviderArn=f'arn:aws:iam::RGW11111111111111111:oidc-provider/{name}.example.com') + + iam.detach_role_policy(RoleName=name, PolicyArn=managed_policy_arn) + iam.delete_role_policy(RoleName=name, PolicyName='Allow') + iam.delete_role(RoleName=name) + + iam.remove_user_from_group(GroupName=name, UserName=name) + iam.detach_group_policy(GroupName=name, PolicyArn=managed_policy_arn) + iam.delete_group_policy(GroupName=name, PolicyName='Allow') + iam.delete_group(GroupName=name) + + iam.detach_user_policy(UserName=name, PolicyArn=managed_policy_arn) + iam.delete_user_policy(UserName=name, PolicyName='Allow') + key_id = iam.list_access_keys(UserName=name)['AccessKeyMetadata'][0]['AccessKeyId'] + iam.delete_access_key(UserName=name, AccessKeyId=key_id) + iam.delete_user(UserName=name) + + realm_meta_checkpoint(realm) + # check that all users/groups/roles are equal across all zones + for source_conn, target_conn in combinations(zonegroup_conns.zones, 2): + if target_conn.zone.has_roles(): + check_roles_eq(source_conn, target_conn) + check_users_eq(source_conn, target_conn) + check_groups_eq(source_conn, target_conn) + check_oidc_providers_eq(source_conn, target_conn)