-
Notifications
You must be signed in to change notification settings - Fork 11
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: check and auto-create S3 artifact bucket if missing (#34)
* feat: check and auto-create S3 artifact bucket if missing Adds the feature that: * Checks related S3 storage for the bucket named by the default_artifact_root config option * If create_artifact_root_if_not_exists==True and default bucket does not exist, attempts to create the bucket Supporting this is an S3 bucket wrapper to help checking the bucket existance/creating new buckets and unit/integration tests. Also included here was some refactoring to make the main function of the Operator read a bit clearer, packaging logic into some verbosely named helpers. Closes #23 * feat: add more unit tests for bucket creation Also refactors/fixes bucket creation logic * fix: typos in services/s3.py, related tests * fix: linting/fmt * refactor: move test_s3.py to unit test folder * feat: add integration test for automatic bucket creation * fix: errors in test_default_bucket_created * fix: linting * fix: integration test for github actions Using AWS client from some virtualized environments prevents the client from inferring the region, resulting in an error. This change sets the region explicitly. * fix: formatting * fix: formatting * Revert "fix: formatting" This reverts commit fbee1a8. * fix: formatting * fix: formatting. again...
- Loading branch information
1 parent
fc8a8da
commit 9430475
Showing
8 changed files
with
581 additions
and
83 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,3 +1,4 @@ | ||
boto3 | ||
ops==1.2.0 | ||
oci-image==1.0.0 | ||
ops-lib-mysql | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,83 @@ | ||
"""Wrapper for basic accessing and validating of S3 Buckets.""" | ||
|
||
import re | ||
from typing import Union | ||
|
||
import boto3 | ||
import botocore.client | ||
import botocore.exceptions | ||
|
||
|
||
class S3BucketWrapper: | ||
"""Wrapper for basic accessing and validating of S3 Buckets.""" | ||
|
||
def __init__( | ||
self, access_key: str, secret_access_key: str, s3_service: str, s3_port: Union[str, int] | ||
): | ||
self.access_key: str = access_key | ||
self.secret_access_key: str = secret_access_key | ||
self.s3_service: str = s3_service | ||
self.s3_port: str = str(s3_port) | ||
|
||
self._client: botocore.client.BaseClient = None | ||
|
||
def check_if_bucket_accessible(self, bucket_name): | ||
"""Checks if a bucket exists and is accessible, returning True if both are satisfied. | ||
Will return False if we encounter a botocore.exceptions.ClientError, which could be | ||
due to the bucket not existing, the client session not having permission to access the | ||
bucket, or some other error with the client. | ||
""" | ||
try: | ||
self.client.head_bucket(Bucket=bucket_name) | ||
return True | ||
except botocore.exceptions.ClientError: | ||
return False | ||
|
||
def create_bucket_if_missing(self, bucket_name): | ||
"""Creates the bucket bucket_name if it does not exist, raising an error if it cannot. | ||
This method tries to access the bucket, assuming that if it is unaccessible that it does | ||
not exist (this is a required assumption as unaccessible buckets look the same as those | ||
that do not exist). If inaccessible, we try to create_bucket and do not catch any | ||
exceptions that result from the call. | ||
""" | ||
if self.check_if_bucket_accessible(bucket_name=bucket_name): | ||
return | ||
|
||
self.create_bucket(bucket_name=bucket_name) | ||
|
||
def create_bucket(self, bucket_name): | ||
"""Create a bucket via the client.""" | ||
self.client.create_bucket(Bucket=bucket_name) | ||
|
||
@property | ||
def client(self) -> botocore.client.BaseClient: | ||
"""Returns an open boto3 client, creating and caching one if needed.""" | ||
if self._client: | ||
return self._client | ||
else: | ||
self._client = boto3.client( | ||
"s3", | ||
endpoint_url=self.s3_url, | ||
aws_access_key_id=self.access_key, | ||
aws_secret_access_key=self.secret_access_key, | ||
) | ||
return self._client | ||
|
||
@property | ||
def s3_url(self): | ||
"""Returns the S3 url.""" | ||
return f"http://{self.s3_service}:{self.s3_port}" | ||
|
||
|
||
def validate_s3_bucket_name(name): | ||
"""Returns True if name is a valid S3 bucket name, else False.""" | ||
# regex from https://stackoverflow.com/a/50484916/5394584 | ||
if re.match( | ||
r"(?=^.{3,63}$)(?!^(\d+\.)+\d+$)(^(([a-z0-9]|[a-z0-9][a-z0-9\-]*[a-z0-9])\.)*([a-z0-9]|[a-z0-9][a-z0-9\-]*[a-z0-9])$)", | ||
name, | ||
): | ||
return True | ||
else: | ||
return False |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,3 +1,5 @@ | ||
black | ||
flake8 | ||
pytest | ||
pytest-mock | ||
pytest-lazy-fixture |
Oops, something went wrong.