DA-1889 Auto create backend bucket (#60)

Author: jessewiles

docs/source/commands.rst

@@ -122,21 +122,6 @@ the current operation.  Acceptable values are: ``gcs`` or ``s3``.
 .. index::
    triple: worker; options; --backend
 
-\\-\\-backend-bucket
-++++++++++++++++++++
-
-The **\\-\\-backend-bucket** option specifies the name of the backend bucket that should
-be used to house the terraform state files.  
-
-.. warning::
-
-   **terraform-worker** does not create the backend bucket. Creation of this file is
-   a prerequistie for running **terraform-worker** with with either a ``gcs`` or
-   ``s3`` backend.
-
-.. index::
-   triple: worker; options; --backend-bucket
-
 .. _backend-prefix:
 
 \\-\\-backend-prefix

docs/source/quick-start.rst

@@ -79,8 +79,7 @@ Next, run the following from a \*nix shell session.
 
 .. note::
    Be sure to replace ``<YOUR_AWS_PROFILE>``, ``<YOUR_BACKEND_BUCKET>``, and ``<YOUR_AWS_REGION>`` with the
-   the appropriate values. Currently **terraform-worker** does not create the bucket.  It will need to be
-   created separately.
+   the appropriate values.
 
 Once the operation is complete, the console should contain text similar to the following:
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "terraform-worker"
-version = "0.10.3"
+version = "0.10.4"
 description = "An orchestration tool for Terraform"
 authors = ["Richard Maynard <richard.maynard@objectrocket.com>"]
 packages = [

tests/conftest.py

@@ -248,7 +248,7 @@ def rootc_options(s3_client, dynamodb_client, sts_client):
 
 
 @pytest.fixture
-def basec(rootc):
+def basec(rootc, s3_client):
     with mock.patch(
         "tfworker.commands.base.BaseCommand.get_terraform_version",
         side_effect=lambda x: (13, 3),
@@ -272,9 +272,12 @@ def gbasec(grootc):
             "tfworker.commands.base.which",
             side_effect=lambda x: "/usr/local/bin/terraform",
         ):
-            return tfworker.commands.base.BaseCommand(
-                grootc, "test-0001", tf_version_major=13
-            )
+            with mock.patch(
+                "tfworker.backends.gcs.storage.Client.from_service_account_json"
+            ):
+                return tfworker.commands.base.BaseCommand(
+                    grootc, "test-0001", tf_version_major=13
+                )
 
 
 @pytest.fixture
@@ -352,12 +355,15 @@ def tf_13cmd_options(rootc_options):
             "tfworker.commands.base.which",
             side_effect=lambda x: "/usr/local/bin/terraform",
         ):
-            return tfworker.commands.terraform.TerraformCommand(
-                rootc_options,
-                deployment="test-0001-options",
-                tf_version=(13, 5),
-                b64_encode=False,
-            )
+            with mock.patch(
+                "tfworker.backends.gcs.storage.Client.from_service_account_json"
+            ):
+                return tfworker.commands.terraform.TerraformCommand(
+                    rootc_options,
+                    deployment="test-0001-options",
+                    tf_version=(13, 5),
+                    b64_encode=False,
+                )
 
 
 @pytest.fixture

tfworker/backends/gcs.py

@@ -17,6 +17,7 @@
 import click
 from google.api_core import page_iterator
 from google.cloud import storage
+from google.cloud.exceptions import Conflict
 
 from .base import BackendError, BaseBackend, validate_backend_empty
 
@@ -38,6 +39,20 @@ def __init__(self, authenticators, definitions, deployment=None):
             if not self._gcs_prefix.endswith("/"):
                 self._gcs_prefix = f"{self._gcs_prefix}/"
 
+            if self._authenticator.creds_path:
+                self._storage_client = storage.Client.from_service_account_json(
+                    self._authenticator.creds_path
+                )
+            else:
+                self._storage_client = storage.Client(
+                    project=self._authenticator.project
+                )
+
+            try:
+                self._storage_client.create_bucket(self._gcs_bucket)
+            except Conflict:
+                pass
+
     def _clean_deployment_limit(self, limit: tuple) -> None:
         """ only clean items within limit """
         full_state_list = self._get_state_list()
@@ -123,16 +138,6 @@ def clean(self, deployment: str, limit: tuple = None) -> None:
         if self._gcs_prefix is None or self._gcs_bucket is None:
             raise BackendError("clean attempted without proper authenticator setup")
 
-        if not hasattr(self, "_storage_client"):
-            if self._authenticator.creds_path:
-                self._storage_client = storage.Client.from_service_account_json(
-                    self._authenticator.creds_path
-                )
-            else:
-                self._storage_client = storage.Client(
-                    project=self._authenticator.project
-                )
-
         # clean entire deployment
         if not limit:
             self._clean_prefix(self._gcs_prefix)

tfworker/backends/s3.py

@@ -16,6 +16,7 @@
 from contextlib import closing
 
 import boto3
+import botocore
 import click
 
 from .base import BackendError, BaseBackend, validate_backend_empty
@@ -55,6 +56,48 @@ def __init__(self, authenticators, definitions, deployment=None):
             )
             self._create_table(locking_table_name)
 
+        # Initialize s3 client and create bucket if necessary. Op should create if
+        # not exists
+        self._s3_client = self._authenticator.backend_session.client("s3")
+        try:
+            self._s3_client.create_bucket(
+                Bucket=self._authenticator.bucket,
+                CreateBucketConfiguration={
+                    "LocationConstraint": self._authenticator.region
+                },
+                ACL="private",
+            )
+        except botocore.exceptions.ClientError as err:
+            err_str = str(err)
+            if "InvalidLocationConstraint" in err_str:
+                click.secho(
+                    "InvalidLocationConstraint raised when trying to create a bucket. "
+                    "Verify the AWS Region passed to the worker matches the AWS region "
+                    "in the profile.",
+                    fg="red",
+                )
+            elif (
+                "BucketAlreadyExists" not in err_str
+                and "BucketAlreadyOwnedByYou" not in err_str
+            ):
+                raise err
+
+        # Block public access
+        self._s3_client.put_public_access_block(
+            Bucket=self._authenticator.bucket,
+            PublicAccessBlockConfiguration={
+                "BlockPublicAcls": True,
+                "IgnorePublicAcls": True,
+                "BlockPublicPolicy": True,
+                "RestrictPublicBuckets": True,
+            },
+        )
+
+        # Enable versioning on the bucket
+        s3_resource = self._authenticator.backend_session.resource("s3")
+        versioning = s3_resource.BucketVersioning(self._authenticator.bucket)
+        versioning.enable()
+
     def _check_table_exists(self, name: str) -> bool:
         """ check if a supplied dynamodb table exists """
         if name in self._ddb_client.list_tables()["TableNames"]:
@@ -69,10 +112,7 @@ def _clean_bucket_state(self, definition=None):
         optionally definition can be passed to limit the cleanup
         to a single definition
         """
-        s3_paginator = self._authenticator.backend_session.client("s3").get_paginator(
-            "list_objects_v2"
-        )
-        s3_client = self._authenticator.backend_session.client("s3")
+        s3_paginator = self._s3_client.get_paginator("list_objects_v2")
 
         if definition is None:
             prefix = self._authenticator.prefix
@@ -82,7 +122,7 @@ def _clean_bucket_state(self, definition=None):
         for s3_object in self.filter_keys(
             s3_paginator, self._authenticator.bucket, prefix
         ):
-            backend_file = s3_client.get_object(
+            backend_file = self._s3_client.get_object(
                 Bucket=self._authenticator.bucket, Key=s3_object
             )
             body = backend_file["Body"]
@@ -149,8 +189,7 @@ def _delete_with_versions(self, key):
         note: in initial testing this isn't required, but is inconsistent with how S3 delete markers, and the boto
         delete object call work there may be some configurations that require extra handling.
         """
-        s3_client = self._authenticator.backend_session.client("s3")
-        s3_client.delete_object(Bucket=self._authenticator.bucket, Key=key)
+        self._s3_client.delete_object(Bucket=self._authenticator.bucket, Key=key)
 
     def clean(self, deployment: str, limit: tuple = None) -> None:
         """