Lana/sc 500507/remove and destroy unattached ebs volumes

.gitignore

@@ -127,3 +127,5 @@ override.tf.json
 # End of https://www.gitignore.io/api/osx,ruby,terraform,visualstudiocode
 
 .kitchen/
+
+.idea/

.gitlab-ci.yml

@@ -0,0 +1,91 @@
+stages:
+  - plan_terraform
+  - apply_terraform
+
+.plan_terraform: &plan_terraform
+  stage: plan_terraform
+  script:
+    - cd ${TF_ENVIRONMENT}
+    - terraform init -no-color
+    - terraform plan -no-color -out=tfplan
+
+  artifacts:
+    untracked: true
+    expire_in: 1 day
+    paths:
+      - ${TF_ENVIRONMENT}/.terraform/**/*
+      - ${TF_ENVIRONMENT}/tfplan
+
+plan_terraform_dev:
+  image: docker.dev.slicelife.com/gitlab-runner-terraform-12:stable
+  <<: *plan_terraform
+  variables:
+    IAM_ROLE: $IAM_ROLE_TERRAFORM_DEV
+    TF_ENVIRONMENT: development
+    VAULT_ADDR: $VAULT_ADDR_DEV
+    VAULT_TOKEN: $VAULT_TOKEN_DEV
+  tags:
+    - development
+
+plan_terraform_prod:
+  image: docker.prod.slicelife.com/gitlab-runner-terraform-12:stable
+  <<: *plan_terraform
+  variables:
+    IAM_ROLE: $IAM_ROLE_TERRAFORM_PROD
+    TF_ENVIRONMENT: production
+    VAULT_ADDR: $VAULT_ADDR_PROD
+    VAULT_TOKEN: $VAULT_TOKEN_PROD
+  script:
+    - cd ${TF_ENVIRONMENT}
+    - terraform init -no-color
+    - terraform taint module.segment.null_resource.segment-setup
+    - terraform plan -no-color -out=tfplan
+  tags:
+    - production
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+    - if: $CI_PIPELINE_SOURCE == "trigger"
+
+.apply_terraform: &apply_terraform
+  stage: apply_terraform
+  script:
+    - cd ${TF_ENVIRONMENT}
+    - terraform apply -no-color tfplan
+
+  artifacts:
+    paths:
+      - ${TF_ENVIRONMENT}/.terraform
+
+apply_terraform_dev:
+  <<: *apply_terraform
+  image: docker.dev.slicelife.com/gitlab-runner-terraform-12:stable
+  variables:
+    IAM_ROLE: $IAM_ROLE_TERRAFORM_DEV
+    TF_ENVIRONMENT: development
+    VAULT_ADDR: $VAULT_ADDR_DEV
+    VAULT_TOKEN: $VAULT_TOKEN_DEV
+  tags:
+    - development
+  rules: 
+    - if: $CI_PIPELINE_SOURCE == "trigger"
+    - when: manual
+  allow_failure: false
+
+apply_terraform_prod:
+  <<: *apply_terraform
+  image: docker.prod.slicelife.com/gitlab-runner-terraform-12:stable
+  variables:
+    IAM_ROLE: $IAM_ROLE_TERRAFORM_PROD
+    TF_ENVIRONMENT: production
+    VAULT_ADDR: $VAULT_ADDR_PROD
+    VAULT_TOKEN: $VAULT_TOKEN_PROD
+  script:
+    - cd ${TF_ENVIRONMENT}
+    - terraform apply -no-color tfplan
+  tags:
+    - production
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "trigger"
+      when: on_success
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+      when: manual

README.md

@@ -1,14 +1,35 @@
-# terraform-aws-data-lake
+# terraform-aws-data-lake (forked)
+
+Forked version of the Segment repo that adds Slice CI/CD.
+
+Updating from the upstream Segment repo to pull in latest changes:
+```bash
+git remote add upstream git@github.com:segmentio/terraform-aws-data-lake.git
+git merge upstream/master
+```
+
+Add the sources in Segment: https://app.segment.com/mypizza-zach/destinations/catalog/data-lakes
+
+| Param        | Value           |
+| ------------ |:--------------|
+| Region: | us-east-1
+| Cluster ID: | j-1T2VIUV1YG249 (ID of the EMR cluster)
+| Glue Catalog | 651565136086
+| Database Name: | segment_data_lake
+| IAM Role: | arn:aws:iam::651565136086:role/segment-data-lake-iam-role
+| S3 Bucket: | 651565136086-slice-segment-data-lake
+
+Note:  **If the Terraform apply recreates the EMR cluster then all Segment destinations wil need to be updated with the new Cluster ID!** 
 
-_Note: Data Lakes is currently in Limited Availability._
+
+# terraform-aws-data-lake
 
 Terraform modules which create AWS resources for a Segment Data Lake.
 
 # Prerequisites
 
-* Accept the [Data Lakes Terms of Service](https://app.segment.com/{workspace_slug}/destinations/catalog?category=DataLakes) (replace the `{workspace_slug}` with your workspace slug).
 * Authorized [AWS account](https://aws.amazon.com/account/).
-* Ability to run Terraform with your AWS Account. You must use Terraform 0.11 or higher.
+* Ability to run Terraform with your AWS Account. Terraform 0.11 and older are supported.
 * A subnet within a VPC for the EMR cluster to run in.
 * An [S3 Bucket](https://github.com/terraform-aws-modules/terraform-aws-s3-bucket) for Segment to load data into. You can create a new one just for this, or re-use an existing one you already have.
 
@@ -57,7 +78,11 @@ mkdir segment-datalakes-tf
 
 ```hcl
 provider "aws" {
-  region = "us-west-2"  # Replace this with the AWS region your infrastructure is set up in.
+  # Replace this with the AWS region your infrastructure is set up in.
+  region = "us-west-2"
+
+  # Currently our modules require the older v2 AWS provider, as upgrading to v3 has notable breaking changes.
+  version = "~> 2"
 }
 
 locals {
@@ -71,26 +96,45 @@ locals {
 
 # This is the target where Segment will write your data.
 # You can skip this if you already have an S3 bucket and just reference that name manually later.
+# If you decide to skip this and use an existing bucket, ensure that you attach a 14 day expiration lifecycle policy to
+# your S3 bucket for the "segment-stage/" prefix.
 resource "aws_s3_bucket" "segment_datalake_s3" {
-  name = "my-first-segment-datalake"
+  bucket = "my-first-segment-datalake"
+
+  lifecycle_rule {
+    enabled = true
+
+    prefix = "segment-stage/"
+
+    expiration {
+      days = 14
+    }
+
+    abort_incomplete_multipart_upload_days = 7
+  }
 }
 
 # Creates the IAM Policy that allows Segment to access the necessary resources
 # in your AWS account for loading your data.
 module "iam" {
-  source = "git@github.com:segmentio/terraform-aws-data-lake//modules/iam?ref=v0.2.0"
+  source = "git@github.com:segmentio/terraform-aws-data-lake//modules/iam?ref=v0.4.0"
+
+  # Suffix is not strictly required if only initializing this module once.
+  # However, if you need to initialize multiple times across different Terraform
+  # workspaces, this hook allows the generated IAM policies to be given unique
+  # names.
+  suffix = "-prod"
 
-  name         = "segment-data-lake-iam-role"
-  s3_bucket    = "${aws_s3_bucket.segment_datalake_s3.name}"
+  s3_bucket    = "${aws_s3_bucket.segment_datalake_s3.id}"
   external_ids = "${values(local.segment_sources)}"
 }
 
 # Creates an EMR Cluster that Segment uses for performing the final ETL on your
 # data that lands in S3.
 module "emr" {
-  source = "git@github.com:segmentio/terraform-aws-data-lake//modules/emr?ref=v0.2.0"
+  source = "git@github.com:segmentio/terraform-aws-data-lake//modules/emr?ref=v0.4.0"
 
-  s3_bucket = "${aws_s3_bucket.segment_datalake_s3.name}"
+  s3_bucket = "${aws_s3_bucket.segment_datalake_s3.id}"
   subnet_id = "subnet-XXX" # Replace this with the subnet ID you want the EMR cluster to run in.
 
   # LEAVE THIS AS-IS
@@ -99,6 +143,7 @@ module "emr" {
   iam_emr_instance_profile = "${module.iam.iam_emr_instance_profile}"
 }
 ```
+
 ## Provision Resources
 * Provide AWS credentials of the account being used. More details here: https://www.terraform.io/docs/providers/aws/index.html
   ```
@@ -180,7 +225,9 @@ If all else fails, teardown and start over.
 
 Terraform 0.11 or higher is supported.
 
-NOTE: Release v0.2.0 onwards only Terraform 0.12 or higher is supported.
+In order to support more versions of Terraform, the AWS Provider needs to held at v2,
+as v3 has breaking changes we don't currently support. Our example `main.tf` has the
+code to accomplish this.
 
 # Development
 
@@ -194,6 +241,24 @@ To develop in this repository, you'll want the following tools set up:
 
 To run unit tests, you also need an AWS account to be able to provision resources.
 
+# Releasing
+
+Releases are made from the master branch. First, make sure you have the last code from master pulled locally:
+
+```
+* git remote update
+* git checkout master
+* git reset origin/master --hard
+```
+
+Then, use [`git release`](https://github.com/tj/git-extras/blob/master/Commands.md#git-release) to cut a new version that follows [semver](https://semver.org):
+
+```
+git release x.y.z
+```
+
+Lastly, craft a new [Github release](https://github.com/segmentio/terraform-aws-data-lake/releases).
+
 # License
 
 Released under the [MIT License](https://opensource.org/licenses/MIT).

development/_backend.tf

@@ -0,0 +1,9 @@
+terraform {
+  backend "s3" {
+    bucket   = "211459479356-terraform-state"
+    key      = "data-engineering/segment-data-lake.tfstate"
+    region   = "us-east-2"
+    role_arn = "arn:aws:iam::211459479356:role/terraform"
+    encrypt  = true
+  }
+}

development/_provider.tf

@@ -0,0 +1,9 @@
+provider "aws" {
+  region  = "us-east-1"
+  profile = "development"
+  version = "2.50"
+}
+
+terraform {
+  required_version = "~> 0.12.0"
+}

development/main.tf

@@ -0,0 +1,69 @@
+locals {
+  s3_bucket_name = "211459479356-slice-segment-data-lake"
+  external_ids = [
+    "KnSuvLUHMG",                         #Direct-Web-QA,
+    "42ukzMtDak",                         #Storefront-QA
+    "e398yK2CXsbPJFbTRU9Nf8"              #Braze-Dev
+  ]                                       # Segment sources that will be enabled for Data Lakes.
+  subnet_id  = "subnet-097e2dc4f7499f77a" # Subnet the EMR cluster will run in.
+  arn_prefix = "arn:aws:iam::211459479356"
+  default_tags = {
+    department = "data"
+    subteam    = "dataeng"
+    git = "https://github.com/slicelife/terraform-aws-data-lake/"
+    environment = "development"
+    terraformed = "yes"
+  }
+}
+
+locals {
+  tags = {
+    s3_bucket_name = "${local.s3_bucket_name}"
+    external_ids   = "${local.external_ids}"
+    subnet_id      = "${local.subnet_id}"
+  }
+}
+
+data "aws_secretsmanager_secret_version" "segment_secrets" {
+  secret_id = "dataeng/segment"
+}
+
+module "s3_bucket" {
+  source    = "../modules/s3_bucket"
+  s3_bucket = local.s3_bucket_name
+  tags = local.default_tags
+  data_account = 409386690817
+}
+
+module "glue" {
+  source = "https://github.com/segmentio/terraform-aws-data-lake/archive/v0.2.0.zip//terraform-segment-data-lakes-0.2.0/modules/glue"
+
+  name = "segment_data_lake"
+}
+
+module "iam" {
+  source = "https://github.com/segmentio/terraform-aws-data-lake/archive/v0.3.0.zip//terraform-segment-data-lakes-0.3.0/modules/iam"
+
+  s3_bucket    = "${local.s3_bucket_name}"
+  external_ids = "${local.external_ids}"
+  tags = local.default_tags
+}
+
+module "emr" {
+  source = "https://github.com/segmentio/terraform-aws-data-lake/archive/v0.3.0.zip//terraform-segment-data-lakes-0.3.0/modules/emr"
+
+  s3_bucket                = "${local.s3_bucket_name}"
+  subnet_id                = "${local.subnet_id}"
+  iam_emr_autoscaling_role = "${local.arn_prefix}:role/${module.iam.iam_emr_autoscaling_role}"
+  iam_emr_service_role     = "${local.arn_prefix}:role/${module.iam.iam_emr_service_role}"
+  iam_emr_instance_profile = "${local.arn_prefix}:instance-profile/${module.iam.iam_emr_instance_profile}"
+  tags = local.default_tags
+}
+
+module "segment" {
+  source      = "../modules/segment"
+  cluster_id  = module.emr.cluster_id
+  environment = "dev"
+  token       = jsondecode(data.aws_secretsmanager_secret_version.segment_secrets.secret_string)["token"]
+  url         = jsondecode(data.aws_secretsmanager_secret_version.segment_secrets.secret_string)["url"]
+}