adding in infrastructure for kairos provisioning and pipelines

integration-framework/moirai-infrastructure/.gitignore

@@ -0,0 +1,9 @@
+**/.terraform
+**/*.tfstate
+
+#Don't check in pem files
+**/*.pem
+.DS_Store
+
+**/kubeconfig_*
+**/secure_vars.yml

integration-framework/moirai-infrastructure/README.md

@@ -0,0 +1,59 @@
+# Application Setup
+
+## Application Deployment
+
+New repo Install order
+
+### Important Configurations (before you get started)
+
+1. all resources generated from this project white list IP addresses for an extra layer of security. Add any corporate/personal IP adresses to allow access terraform\environment-tf\variables.tf 
+
+2. If using the Docker-repo-tf set the registry admin password insid terraform\docker-repo-tf\scripts\docker-registry.sh
+
+3. For Zeus (Beachhead) application the Relational Database Server password needs to be set. There are two different files, differing in the tier (i.e. Test, Stage, Production)
+    * For lower tiers (Test, Staging) set the variable default 'db_secret_lower_tier' in terraform\rds-lower-tiers-tf\variables.tf
+    * For production tier set the variable default 'db_secret' in terraform\rds-tf\variables.tf
+
+4. In ansible/group_vars/vars_file.yml set any missing variables such as cert_email. These need to be filled out before the scripts can run
+
+### How to setup your local Unix environment setup. (You probably need this)
+
+1. First you need to create an account in aws that has acccess to run scripts, preferably an admin. Setup awscli locally with the credential keys for this user under the profile name "kairos"
+    "kairos" is the profile many of the scripts are set to use.
+
+2. install mysql-client (not mandatory but useful)
+
+3. Install ansible by running 'install/install_ansible.sh' (This includes python3, pip3, gcc, epel-release)
+
+4. Install terraform by running 'install/install_terraform.sh' It is imperative that you use the version in the script!
+
+4. Run 'ansible-playbook -i inventory kairos-keys.yml' 
+
+5. You also need to install jq to run the k8s stuff, there is no script for this yet
+
+
+### How to setup the aws environment (You probably don't need this, this is once per project).
+
+Locally run the following...
+
+1. Run 'ansible/kairos-infrastructure.yml' either using the full command 'ansible-playbook -i inventory kairos-infrastructure.yml' or simply './kairos-infrastructure.yml'
+    This script create the s3 buckets to store terraform states.
+
+2. Run 'ansible/kairos-keys.yml' either using the full command 'ansible-playbook -i inventory kairos-keys.yml' or simply './kairos-keys.yml'
+
+2. Go to environment-tf, run 'terraform init' and 'terraform apply'
+    This terraform scripts sets up all the roles/policies, security groups, additional s3 buckets
+    * Note: To add personal IP addresses modify 'employee_cidrs' in the variables-tf file
+    * Note: To add work related IP addresses modify 'employee_cidrs' in the variables-tf file
+
+3. Go to rds-tf, run 'terraform init' and 'terraform apply' (Production)
+    This terraform script will setup the database used for the main Zeus application to store information
+
+4. Go to rds-lower-tiers, run 'terraform init' and 'terraform apply'
+    This terraform script will setup the database used for the main Zeus application to store informatio
+
+5. Go to beachhead-tf, run 'terraform init' and 'terraform apply'
+    This terraform script sets up our beachhead which 
+    1. sets up the environment for services
+    2. starts zeus and ui applications
+

integration-framework/moirai-infrastructure/ansible/beachhead-setup-ecr.yml

@@ -0,0 +1,23 @@
+
+---
+- name: Wait for systems to come up
+  hosts: localhost
+  connection: local
+  gather_facts: true
+  roles:
+    - { role: aws-ecr-checkout, aws_profile: kairos, repo_name: zeus, repo_tag: latest, local_name: zeus, local_tag: temp}
+    - { role: aws-ecr-checkin, aws_profile: kairos, repo_name: genesis, repo_tag: zeus-latest, local_name: zeus, local_tag: temp}
+    - { role: aws-ecr-checkout, aws_profile: kairos, repo_name: clotho, repo_tag: latest, local_name: clotho, local_tag: temp}
+    - { role: aws-ecr-checkin, aws_profile: kairos, repo_name: genesis, repo_tag: clotho-latest, local_name: clotho, local_tag: temp}
+    - { role: aws-ecr-checkout, aws_profile: kairos, repo_name: ui, repo_tag: latest, local_name: ui, local_tag: temp}
+    - { role: aws-ecr-checkin, aws_profile: kairos, repo_name: genesis, repo_tag: ui-latest, local_name: ui, local_tag: temp}
+
+- name: Clean up
+  hosts: localhost
+  connection: local
+  gather_facts: true
+  tasks:
+    - shell: docker rmi zeus:temp
+    - shell: docker rmi clotho:temp
+    - shell: docker rmi ui:temp
+...

integration-framework/moirai-infrastructure/ansible/bounce_validation_external.yml

@@ -0,0 +1,44 @@
+#!/usr/bin/env ansible-playbook -i inventory
+
+---
+- name: Wait for systems to come up
+  hosts: localhost
+  connection: local
+  gather_facts: true
+  roles:
+    - {role: wait-for-ec2,
+        tags_to_check: ["tag_Name_Validation_External"]}
+
+- name: Add Clotho
+  gather_facts: true
+  hosts: 
+    - "tag_Name_Validation_External"
+  remote_user: ubuntu
+  become: true
+  vars:
+    db_type: "Validation"
+    clotho_tag: "latest"
+    network_name: 'nginx-network'
+    NEPTUNE_DB_URI: ""
+  roles:
+    - role: aws-ecr-checkout
+      repo_name: clotho 
+      repo_tag: "{{ clotho_tag|lower }}"
+      aws_profile: ''
+    # Start Clotho app
+    - role: bounce-container
+      container_name: validation
+      image_name: clotho
+      image_tag: "{{ clotho_tag|lower }}"
+      restart: "always"
+      ports:
+        - "8008"
+      env: {
+        DB_TYPE: "{{ db_type }}",
+        DB_URI: "{{ NEPTUNE_DB_URI }}",
+        DB_PORT: "8182",
+        AWS_ACCESS_KEY_ID: "{{ lookup('env','aws_access_key') }}",
+        AWS_SECRET_ACCESS_KEY: "{{ lookup('env','aws_secret_key') }}",
+        SERVICE_REGION: "us-east-1"
+      }
+    - { role: kairos-proxy, conf_version: shared }

integration-framework/moirai-infrastructure/ansible/bounce_validation_internal.yml

@@ -0,0 +1,36 @@
+#!/usr/bin/env ansible-playbook -i inventory
+
+---
+- name: Wait for systems to come up
+  hosts: localhost
+  connection: local
+  gather_facts: true
+  roles:
+    - {role: wait-for-ec2,
+        tags_to_check: ["tag_Name_Validation_Internal"]}
+
+- name: Add Clotho
+  gather_facts: true
+  hosts: 
+    - "tag_Name_Validation_Internal"
+  remote_user: ubuntu
+  become: true
+  vars:
+    db_type: "Validation"
+    clotho_tag: "latest"
+  roles:
+    - role: aws-ecr-checkout
+      repo_name: clotho 
+      repo_tag: "{{ clotho_tag|lower }}"
+      aws_profile: ''
+    # Start Clotho app
+    - role: bounce-container
+      container_name: validation
+      image_name: clotho
+      image_tag: "{{ clotho_tag|lower }}"
+      restart: "always"
+      ports:
+        - "80:8008"
+      env: {
+        DB_TYPE: "{{ db_type }}"
+      }

integration-framework/moirai-infrastructure/ansible/deploy-dev-box.yml

@@ -0,0 +1,22 @@
+---
+- name: Configure the instances with common requirements
+  hosts: localhost
+  gather_facts: true
+  remote_user: local
+  become: true
+  roles:
+    - docker
+    - docker-clean
+    - docker-clean-images-crontab
+    - aws-ssm
+    - awscli
+    - pip3-docker
+    - pip-botos
+- name: set facts
+  hosts: localhost
+  gather_facts: true
+  remote_user: local
+  become: true
+  tasks:
+    - set_fact:
+        container_network_name: "nginx-network"

integration-framework/moirai-infrastructure/ansible/deploy-docker-repository-b.yml

@@ -0,0 +1,9 @@
+---
+
+- name: Start Containers
+  gather_facts: true
+  hosts: localhost
+  remote_user: local
+  become: true
+  roles:
+    - {role: kairos-proxy, conf_version: docker}

integration-framework/moirai-infrastructure/ansible/deploy-docker-repository-ssl.yml

@@ -0,0 +1,9 @@
+---
+
+- name: obtain all information for a single ACM certificate
+  gather_facts: true
+  hosts: localhost
+  remote_user: local
+  become: true
+  roles:
+    - {role: ssl-cert, ec2_cname: docker}

integration-framework/moirai-infrastructure/ansible/deploy-docker-repository.yml

@@ -0,0 +1,48 @@
+---
+- name: Configure the instances with common requirements
+  hosts: localhost
+  gather_facts: true
+  remote_user: local
+  become: true
+  roles:
+    - docker
+    - docker-clean
+    # - docker-clean-images-crontab
+    - aws-ssm
+    - awscli
+    - pip3-docker
+    - pip-botos
+
+- name: set facts
+  hosts: localhost
+  gather_facts: true
+  remote_user: local
+  become: true
+  tasks:
+    - set_fact:
+        container_network_name: "nginx-network"
+
+- name: obtain all information for a single ACM certificate
+  gather_facts: true
+  hosts: localhost
+  remote_user: local
+  become: true
+  roles:
+    - {role: flask-docker-registry-service}
+    - {role: ssl-cert, ec2_cname: docker}
+
+- name: Create docker network
+  gather_facts: true
+  hosts: localhost
+  remote_user: local
+  become: true
+  roles:
+    - { role: docker-network, network_name: "{{ container_network_name }}"}
+
+- name: Start Containers
+  gather_facts: true
+  hosts: localhost
+  remote_user: local
+  become: true
+  roles:
+    - {role: kairos-proxy, conf_version: docker}

integration-framework/moirai-infrastructure/ansible/deploy-proxy-docker.yml

@@ -0,0 +1,13 @@
+#!/usr/bin/env ansible-playbook -i inventory
+
+---
+- name: Create network
+  gather_facts: true
+  hosts: localhost
+  remote_user: local
+  become: true
+  roles:
+    # Retrieve images from ECR via AWS-CLI
+    - { role: neptune-proxy, network_name: proxy_net}
+
+...

integration-framework/moirai-infrastructure/ansible/deploy-zeus-dockers.yml

@@ -0,0 +1,62 @@
+
+---
+
+- name: Pull docker containers
+  gather_facts: true
+  hosts: localhost
+  remote_user: local
+  become: true
+  roles:
+    - { role: aws-ecr-checkout, aws_profile: null, repo_name: genesis, repo_tag: zeus-latest, local_name: zeus, local_tag: latest}
+    - { role: aws-ecr-checkout, aws_profile: null, repo_name: genesis, repo_tag: ui-latest, local_name: ui, local_tag: latest}
+
+- name: Start Containers
+  gather_facts: true
+  hosts: localhost
+  remote_user: local
+  become: true
+  vars:
+    network_name: beach_net
+    rds_address_env: "{{ lookup('env','rds_address') }}"
+    rds_port_env: "{{ lookup('env','rds_port') }}"
+    rds_db_name_env: "{{ lookup('env','rds_database_name') }}"
+    rds_connector: "jdbc:mysql://{{ rds_address_env }}:{{ rds_port_env }}/{{ rds_db_name_env }}"
+    aws_region: "us-east-1"
+    env: {
+      SPRING_DATASOURCE_URL: "{{ rds_connector }}",
+      SPRING_DATASOURCE_USERNAME: "{{ lookup('env','rds_username') }}",
+      SPRING_DATASOURCE_PASSWORD: "{{ lookup('env','rds_secret') }}",
+      AWS_REGION: "{{ aws_region }}",
+      ANSIBLE_HOST_KEY_CHECKING: "false",
+      AWS_ACCESS_KEY_ID: "{{ lookup('env','aws_access_key') }}",
+      AWS_SECRET_ACCESS_KEY: "{{ lookup('env','aws_secret_key') }}",
+      ENVIRONMENT_TIER: "{{ lookup('env','environment_tier_zeus') }}",
+      MAIL_USERNAME: "{{ lookup('env','mail_username') }}",
+      MAIL_PASSWORD: "{{ lookup('env','mail_password') }}",
+      SG_AWS_ID_ENV: "{{ lookup('env','sg_aws_id') }}",
+      SUBNET_ID_ENV: "{{ lookup('env','subnet_id') }}"
+    }
+  roles:
+    # Start application
+    - role: bounce-container
+      restart: "always"
+      container_name: "zeus-latest"
+      image_name: "zeus"
+      ports:
+        - "8000:8000"
+      pause_time: 2
+      volumes:
+        - /home/ubuntu/moirai-infrastructure/:/usr/src/moirai-infrastructure/
+        - /home/ubuntu/moirai-infrastructure/ansible:/usr/src/ansible
+        - /home/ubuntu/moirai-infrastructure/key-pairs:/usr/src/key-pairs
+        - /home/ubuntu/.aws:/root/.aws
+    - role: bounce-container
+      restart: "always"
+      container_name: "ui-latest"
+      image_name: "ui"
+      ports:
+        - "all"
+    # Must run after ui bounce container
+    - {role: kairos-proxy, conf_version: production}
+    - role: flask-prod-deploy-service
+...