Skip to content

Commit

Permalink
rgw/sfs: Implement new versioning design
Browse files Browse the repository at this point in the history 
Implements new versioning design.

Includes `rgw::sal::Attrs` type binding in `sqliteorm` code, so
VersionedObject code for conversions is no longer needed.

Adds transactions for the actions that require more than 1 database
access.

Simplifies the way a new version is created and moves its logic to a
sqlite transaction to avoid race conditions when creating N objects with
the same name at once from several threads.

Fixes: https://github.com/aquarist-labs/s3gw/issues/378
Fixes: https://github.com/aquarist-labs/s3gw/issues/472
Fixes: https://github.com/aquarist-labs/s3gw/issues/547
Fixes: https://github.com/aquarist-labs/s3gw/issues/526
Fixes: https://github.com/aquarist-labs/s3gw/issues/524
Fixes: https://github.com/aquarist-labs/s3gw/issues/519

Signed-off-by: Xavi Garcia <xavi.garcia@suse.com>
  • Loading branch information
@0xavi0
0xavi0 committed Jun 8, 2023
1 parent 3db8861 commit a32f63e
Show file tree
Hide file tree
Showing 34 changed files with 2,040 additions and 955 deletions.
50 changes: 25 additions & 25 deletions qa/rgw/store/sfs/tests/fixtures/s3-tests.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ test_bucket_listv2_fetchowner_defaultempty
test_bucket_listv2_fetchowner_empty
test_bucket_list_delimiter_not_exist
test_bucket_listv2_delimiter_not_exist
# test_bucket_list_delimiter_not_skip_special
test_bucket_list_delimiter_not_skip_special
test_bucket_list_prefix_basic
test_bucket_listv2_prefix_basic
test_bucket_list_prefix_alt
Expand Down Expand Up @@ -78,7 +78,7 @@ test_bucket_listv2_startafter_unreadable
# test_bucket_list_marker_after_list
# test_bucket_listv2_startafter_after_list
# test_bucket_list_return_data
# test_bucket_list_return_data_versioning
test_bucket_list_return_data_versioning
test_bucket_list_objects_anonymous
test_bucket_listv2_objects_anonymous
test_bucket_list_objects_anonymous_fail
Expand All @@ -92,10 +92,10 @@ test_object_write_to_nonexist_bucket
test_bucket_create_delete
test_object_read_not_exist
# test_object_requestid_matches_header_on_error
# test_multi_object_delete
# test_multi_objectv2_delete
# test_multi_object_delete_key_limit
# test_multi_objectv2_delete_key_limit
test_multi_object_delete
test_multi_objectv2_delete
test_multi_object_delete_key_limit
test_multi_objectv2_delete_key_limit
test_object_head_zero_bytes
test_object_write_check_etag
test_object_write_cache_control
Expand Down Expand Up @@ -252,7 +252,7 @@ test_bucket_create_naming_good_starts_alpha
test_bucket_create_naming_good_starts_digit
test_bucket_create_naming_good_contains_period
test_bucket_create_naming_good_contains_hyphen
# test_bucket_recreate_not_overriding
test_bucket_recreate_not_overriding
# test_bucket_create_special_key_names
test_bucket_list_special_prefix
test_object_copy_zero_size
Expand All @@ -269,9 +269,9 @@ test_object_copy_retaining_metadata
# test_object_copy_replacing_metadata
test_object_copy_bucket_not_found
test_object_copy_key_not_found
# test_object_copy_versioned_bucket
test_object_copy_versioned_bucket
test_object_copy_versioned_url_encoding
# test_object_copy_versioning_multipart_upload
test_object_copy_versioning_multipart_upload
test_multipart_upload_empty
# test_multipart_upload_small
# test_multipart_copy_small
Expand All @@ -280,7 +280,7 @@ test_multipart_copy_improper_range
# test_multipart_copy_without_range
# test_multipart_copy_special_names
# test_multipart_upload
# test_multipart_copy_versioned
test_multipart_copy_versioned
# test_multipart_upload_resend_part
test_multipart_upload_multiple_sizes
# test_multipart_copy_multiple_sizes
Expand Down Expand Up @@ -320,24 +320,24 @@ test_ranged_request_return_trailing_bytes_response_code
test_ranged_request_invalid_range
test_ranged_request_empty_object
test_versioning_bucket_create_suspend
# test_versioning_obj_create_read_remove
# test_versioning_obj_create_read_remove_head
test_versioning_obj_create_read_remove
test_versioning_obj_create_read_remove_head
test_versioning_obj_plain_null_version_removal
# test_versioning_obj_plain_null_version_overwrite
test_versioning_obj_plain_null_version_overwrite_suspended
# test_versioning_obj_suspend_versions
# test_versioning_obj_create_versions_remove_all
# test_versioning_obj_create_versions_remove_special_names
# test_versioning_obj_create_overwrite_multipart
# test_versioning_obj_list_marker
# test_versioning_copy_obj_version
# test_versioning_multi_object_delete
# test_versioning_multi_object_delete_with_marker
# test_versioning_multi_object_delete_with_marker_create
# test_versioned_object_acl
# test_versioned_object_acl_no_version_specified
# test_versioned_concurrent_object_create_concurrent_remove
# test_versioned_concurrent_object_create_and_remove
test_versioning_obj_create_versions_remove_all
test_versioning_obj_create_versions_remove_special_names
test_versioning_obj_create_overwrite_multipart
test_versioning_obj_list_marker
test_versioning_copy_obj_version
test_versioning_multi_object_delete
test_versioning_multi_object_delete_with_marker
test_versioning_multi_object_delete_with_marker_create
test_versioned_object_acl
test_versioned_object_acl_no_version_specified
test_versioned_concurrent_object_create_concurrent_remove
test_versioned_concurrent_object_create_and_remove
test_lifecycle_set
test_lifecycle_get
test_lifecycle_get_no_id
Expand Down Expand Up @@ -455,7 +455,7 @@ test_object_lock_put_obj_lock_invalid_bucket
# test_object_lock_put_obj_lock_invalid_years
# test_object_lock_put_obj_lock_invalid_mode
# test_object_lock_put_obj_lock_invalid_status
# test_object_lock_suspend_versioning
test_object_lock_suspend_versioning
# test_object_lock_get_obj_lock
test_object_lock_get_obj_lock_invalid_bucket
# test_object_lock_put_obj_retention
Expand Down
18 changes: 5 additions & 13 deletions qa/rgw/store/sfs/tests/test-sfs-lifecycle-smoke.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,17 +130,13 @@ def test_expiration(self):
{
"ID": "rule1",
"Expiration": {"Days": 1},
"Filter": {
"Prefix": "expire1/"
},
"Filter": {"Prefix": "expire1/"},
"Status": "Enabled",
},
{
"ID": "rule2",
"Expiration": {"Days": 5},
"Filter": {
"Prefix": "expire3/"
},
"Filter": {"Prefix": "expire3/"},
"Status": "Enabled",
},
]
Expand Down Expand Up @@ -200,9 +196,7 @@ def test_lifecycle_versioning_enabled(self):
{
"ID": "rule1",
"Expiration": {"Days": 1},
"Filter": {
"Prefix": "expire1/"
},
"Filter": {"Prefix": "expire1/"},
"Status": "Enabled",
}
]
Expand All @@ -212,7 +206,7 @@ def test_lifecycle_versioning_enabled(self):
)

# give enough time to expire.
# 4 cycles because:
# 3 cycles because:
# 1st cycle won't be expired yet (not still 1 day)
# 2nd cycle rgw considers the bucket at processed
# today and skips it
Expand All @@ -238,9 +232,7 @@ def test_expiration_multiple_buckets(self):
{
"ID": "rule1",
"Expiration": {"Days": 1},
"Filter": {
"Prefix": "expire1/"
},
"Filter": {"Prefix": "expire1/"},
"Status": "Enabled",
}
]
Expand Down
1 change: 1 addition & 0 deletions qa/rgw/store/sfs/tests/test-sfs-object-locking.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -462,6 +462,7 @@ def test_object_locking_legal_hold(self):

self.assertTrue(response["ResponseMetadata"]["HTTPStatusCode"] == 204)


if __name__ == "__main__":
if len(sys.argv) == 2:
address_port = sys.argv.pop()
Expand Down
Loading

0 comments on commit a32f63e

Please sign in to comment.