build the provider in github

.ci/scripts/build-environment/downstream-builder/generate_downstream.sh

@@ -82,9 +82,6 @@ clone_repo
 git config --local user.name "Modular Magician"
 git config --local user.email "magic-modules@google.com"
 
-# MMv1 now lives inside a sub-folder
-pushd mmv1
-
 if [ "$COMMAND" == "head" ]; then
     BRANCH=auto-pr-$REFERENCE
     COMMIT_MESSAGE="New generated code for MM PR $REFERENCE."
@@ -101,67 +98,90 @@ elif [ "$COMMAND" == "downstream" ]; then
     COMMIT_MESSAGE="$ORIGINAL_MESSAGE$NEWLINE[upstream:$REFERENCE]"
 fi
 
-if [ "$REPO" == "terraform" ]; then
-    pushd $LOCAL_PATH
-    go mod download
-    find . -type f -not -wholename "./.git*" -not -wholename "./.changelog*" -not -name ".travis.yml" -not -name ".golangci.yml" -not -name "CHANGELOG.md" -not -name "CHANGELOG_v*.md" -not -name "GNUmakefile" -not -name "docscheck.sh" -not -name "LICENSE" -not -name "README.md" -not -wholename "./examples*" -not -name ".go-version" -not -name ".hashibot.hcl" -print0 | xargs -0 git rm
-    popd
-fi
+if [ "$COMMAND" == "base" ]; then
+    pushd mmv1
+    if [ "$REPO" == "terraform" ]; then
+        pushd $LOCAL_PATH
+        go mod download
+        find . -type f -not -wholename "./.git*" -not -wholename "./.changelog*" -not -name ".travis.yml" -not -name ".golangci.yml" -not -name "CHANGELOG.md" -not -name "CHANGELOG_v*.md" -not -name "GNUmakefile" -not -name "docscheck.sh" -not -name "LICENSE" -not -name "README.md" -not -wholename "./examples*" -not -name ".go-version" -not -name ".hashibot.hcl" -print0 | xargs -0 git rm
+        popd
+    fi
 
-if [ "$REPO" == "terraform-google-conversion" ]; then
-    # Generate tfplan2cai
-    pushd $LOCAL_PATH
-    # clear out the templates as they are copied during
-    # generation from mmv1/third_party/validator/tests/data
-    rm -rf ./tfplan2cai/testdata/templates/
-    rm -rf ./tfplan2cai/testdata/generatedconvert/
-    rm -rf ./tfplan2cai/converters/google/provider
-    rm -rf ./tfplan2cai/converters/google/resources
-    find ./tfplan2cai/test/** -type f -exec git rm {} \;
-    popd
+    if [ "$REPO" == "terraform-google-conversion" ]; then
+        # Generate tfplan2cai
+        pushd $LOCAL_PATH
+        # clear out the templates as they are copied during
+        # generation from mmv1/third_party/validator/tests/data
+        rm -rf ./tfplan2cai/testdata/templates/
+        rm -rf ./tfplan2cai/testdata/generatedconvert/
+        rm -rf ./tfplan2cai/converters/google/provider
+        rm -rf ./tfplan2cai/converters/google/resources
+        find ./tfplan2cai/test/** -type f -exec git rm {} \;
+        popd
 
-    bundle exec compiler.rb -a -e terraform -f validator -o $LOCAL_PATH/tfplan2cai -v $VERSION
+        bundle exec compiler.rb -a -e terraform -f validator -o $LOCAL_PATH/tfplan2cai -v $VERSION
 
-    # Generate cai2hcl
-    pushd $LOCAL_PATH
-    rm -rf ./cai2hcl/*
-    popd
+        # Generate cai2hcl
+        pushd $LOCAL_PATH
+        rm -rf ./cai2hcl/*
+        popd
 
-    bundle exec compiler.rb -a -e terraform -f tgc_cai2hcl -o $LOCAL_PATH/cai2hcl -v $VERSION
+        bundle exec compiler.rb -a -e terraform -f tgc_cai2hcl -o $LOCAL_PATH/cai2hcl -v $VERSION
 
-    if [ "$COMMAND" == "downstream" ]; then
-      pushd $LOCAL_PATH
-      go get -d github.com/hashicorp/terraform-provider-google-beta@$BASE_BRANCH
-      go mod tidy
-      set +e
-      make build
-      set -e
-      popd
+        if [ "$COMMAND" == "downstream" ]; then
+          pushd $LOCAL_PATH
+          go get -d github.com/hashicorp/terraform-provider-google-beta@$BASE_BRANCH
+          go mod tidy
+          set +e
+          make build
+          set -e
+          popd
+        fi
+    elif [ "$REPO" == "tf-oics" ]; then
+        # use terraform generator with oics override
+        bundle exec compiler.rb -a -e terraform -f oics -o $LOCAL_PATH -v $VERSION
+    else
+        if [ "$REPO" == "terraform" ]; then
+            if [ "$VERSION" == "ga" ]; then
+                bundle exec compiler.rb -a -e $REPO -o $LOCAL_PATH -v $VERSION --no-docs
+                bundle exec compiler.rb -a -e $REPO -o $LOCAL_PATH -v beta --no-code
+            else
+                bundle exec compiler.rb -a -e $REPO -o $LOCAL_PATH -v $VERSION
+            fi
+            pushd ../
+            make tpgtools OUTPUT_PATH=$LOCAL_PATH VERSION=$VERSION
+
+            # Only generate TeamCity-related file for TPG and TPGB
+            if [ "$VERSION" == "ga" ] || [ "$VERSION" == "beta" ]; then
+                make teamcity-servicemap-generate OUTPUT_PATH=$LOCAL_PATH VERSION=$VERSION
+            fi
+            popd
+        fi
     fi
-elif [ "$REPO" == "tf-oics" ]; then
-    # use terraform generator with oics override
-    bundle exec compiler.rb -a -e terraform -f oics -o $LOCAL_PATH -v $VERSION
+    popd
 else
-    if [ "$REPO" == "terraform" ]; then
-        if [ "$VERSION" == "ga" ]; then
-            bundle exec compiler.rb -a -e $REPO -o $LOCAL_PATH -v $VERSION --no-docs
-            bundle exec compiler.rb -a -e $REPO -o $LOCAL_PATH -v beta --no-code
-        else
-            bundle exec compiler.rb -a -e $REPO -o $LOCAL_PATH -v $VERSION
+    if [ "$REPO" == "terraform-google-conversion" ]; then
+        make clean-tgc OUTPUT_PATH="$LOCAL_PATH"
+        make tgc OUTPUT_PATH="$LOCAL_PATH"
+
+        if [ "$COMMAND" == "downstream" ]; then
+          pushd $LOCAL_PATH
+          go get -d github.com/hashicorp/terraform-provider-google-beta@$BASE_BRANCH
+          go mod tidy
+          set +e
+          make build
+          set -e
+          popd
         fi
-        pushd ../
-        make tpgtools OUTPUT_PATH=$LOCAL_PATH VERSION=$VERSION
-
-        # Only generate TeamCity-related file for TPG and TPGB
-        if [ "$VERSION" == "ga" ] || [ "$VERSION" == "beta" ]; then
-            make teamcity-servicemap-generate OUTPUT_PATH=$LOCAL_PATH VERSION=$VERSION
-        fi
-        popd
+    elif [ "$REPO" == "tf-oics" ]; then
+        # use terraform generator with oics override
+        make tf-oics OUTPUT_PATH="$LOCAL_PATH"
+    elif [ "$REPO" == "terraform" ]; then
+        make clean-provider OUTPUT_PATH="$LOCAL_PATH"
+        make provider OUTPUT_PATH="$LOCAL_PATH" VERSION=$VERSION
     fi
 fi
 
-popd
-
 pushd $LOCAL_PATH
 
 git config --local user.name "Modular Magician"
@@ -222,5 +242,3 @@ if [ "$COMMITTED" == "true" ] && [ "$COMMAND" == "downstream" ]; then
         -d '{"merge_method": "squash"}' \
         "https://api.github.com/repos/$UPSTREAM_OWNER/$GH_REPO/pulls/$NEW_PR_NUMBER/merge"
 fi
-
-popd

.github/workflows/build-downstream.yml

@@ -0,0 +1,114 @@
+name: build
+permissions: read-all
+
+on:
+  workflow_call:
+    inputs:
+      repo:
+        description: 'Repository to build'
+        required: true
+        type: string
+
+concurrency:
+  group: ${{ inputs.repo  }}-${{ github.event_name == 'pull_request' && format('pr-{0}-', github.event.pull_request.number) || format('commit-{0}', github.sha) }}
+  cancel-in-progress: true
+
+jobs:
+  generate-repository:
+    runs-on: ubuntu-22.04
+    env:
+      BASE_BRANCH: ${{ github.event.pull_request.base.ref || github.ref }}
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@ec02537da5712d66d4d50a0f33b7eb52773b5ed1
+        with:
+          ruby-version: '3.1'
+
+      - name: Cache Bundler gems
+        uses: actions/cache@v2
+        with:
+          path: mmv1/vendor/bundle
+          key: ${{ runner.os }}-gems-${{ hashFiles('mmv1/**/Gemfile.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-gems-
+
+      - name: Install Ruby dependencies
+        run: |
+          bundle config path mmv1/vendor/bundle
+          bundle install
+        working-directory: mmv1
+
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: '^1.19'
+
+      # Cache Go modules
+      - name: Cache Go modules
+        uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+
+      - run: go install golang.org/x/tools/cmd/goimports@latest
+
+      - name: Build ${{ inputs.repo  }}
+        run: |
+          set -e
+          set -x
+          # Set GOPATH to a directory the runner user has access to
+          export GOPATH=~/go
+
+          function clone_repo() {
+            export OUTPUT_PATH=$GOPATH/src/github.com/$UPSTREAM_OWNER/$GH_REPO
+            GITHUB_PATH=https://x-access-token:$GITHUB_TOKEN@github.com/$UPSTREAM_OWNER/$GH_REPO
+            mkdir -p "$(dirname $OUTPUT_PATH)"
+            git clone $GITHUB_PATH $OUTPUT_PATH --branch $BASE_BRANCH
+          }
+
+          GH_REPO="${{ inputs.repo  }}"
+          if [ "$GH_REPO" == "docs-examples" ] && [ "$BASE_BRANCH" == "main" ]; then
+              BASE_BRANCH="master"
+          fi
+
+          GITHUB_PATH=https://x-access-token:$GITHUB_TOKEN@github.com/$UPSTREAM_OWNER/$GH_REPO
+
+          if [[ "$GH_REPO" == terraform-provider-google* ]]; then
+            UPSTREAM_OWNER=hashicorp
+            clone_repo
+          if [ "$GH_REPO" == "terraform-provider-google" ]; then
+              export VERSION=ga
+            else
+              export VERSION=beta
+            fi
+            make clean-provider
+            make provider
+          elif [ "$GH_REPO" == "terraform-google-conversion" ]; then
+            UPSTREAM_OWNER=GoogleCloudPlatform
+            clone_repo
+            make clean-tgc
+            make tgc
+          elif [ "$GH_REPO" == "docs-examples" ]; then
+            UPSTREAM_OWNER=terraform-google-modules
+            clone_repo
+            make tf-oics
+          else
+            echo "case not supported"
+            exit 1
+          fi
+
+          (current_dir=$(pwd) && cd $OUTPUT_PATH && zip -r "$current_dir/output.zip" .)
+
+      - name: Upload built artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: artifact-${{ inputs.repo  }}
+          path: output.zip

.github/workflows/downstreams.yml

@@ -0,0 +1,50 @@
+name: downstreams
+permissions: read-all
+
+on:
+  push:
+    branches:
+      - main
+      - 'FEATURE-BRANCH-*'
+
+concurrency:
+  group: ${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || format('commit-{0}', github.sha) }}
+  cancel-in-progress: true
+
+jobs:
+
+  terraform-provider-google:
+    uses: ./.github/workflows/build-downstream.yml
+    with:
+      repo: 'terraform-provider-google'
+  terraform-provider-google-beta:
+    uses: ./.github/workflows/build-downstream.yml
+    with:
+      repo: 'terraform-provider-google-beta'
+  terraform-google-conversion:
+    uses: ./.github/workflows/build-downstream.yml
+    with:
+      repo: 'terraform-google-conversion'
+  docs-examples:
+    uses: ./.github/workflows/build-downstream.yml
+    with:
+      repo: 'docs-examples'
+
+  unit-test-terraform-provider-google:
+    name: terraform-provider-google
+    needs: terraform-provider-google
+    uses: ./.github/workflows/unit-test-tpg.yml
+    with:
+      repo: 'terraform-provider-google'
+
+  unit-test-terraform-provider-googele-beta:
+    name: terraform-provider-google-beta
+    needs: terraform-provider-google-beta
+    uses: ./.github/workflows/unit-test-tpg.yml
+    with:
+      repo: 'terraform-provider-google-beta'
+
+  unit-test-terraform-google-conversion:
+    name: terraform-google-conversion
+    needs: [terraform-google-conversion, terraform-provider-google-beta]
+    uses: ./.github/workflows/unit-test-tgc.yml

.github/workflows/unit-test-tgc.yml

@@ -0,0 +1,55 @@
+name: unit-test-tgc
+permissions: read-all
+
+on:
+  workflow_call:
+
+jobs:
+  compile-and-unit-tests:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Download built artifacts
+        uses: actions/download-artifact@v2
+        with:
+          name: artifact-terraform-google-conversion
+          path: artifacts-tgc
+
+      - name: Download built artifacts
+        uses: actions/download-artifact@v2
+        with:
+          name: artifact-terraform-provider-google-beta
+          path: artifacts-tpgb
+
+      - name: Unzip the artifacts and delete the zip
+        run: |
+          unzip artifacts-tgc/output.zip -d ./tgc
+          unzip artifacts-tpgb/output.zip -d ./tpgb
+          rm artifacts-tgc/output.zip
+          rm artifacts-tpgb/output.zip
+
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: '^1.19'
+
+      - name: Cache Go modules and build cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-test-terraform-google-conversion-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-test-terraform-google-conversion-${{ hashFiles('**/go.sum') }}
+            ${{ runner.os }}-test-terraform-google-conversion-
+
+      - name: Build Terraform Google Conversion
+        run: |
+          cd tgc
+          go mod edit -replace=github.com/hashicorp/terraform-provider-google-beta=../tpgb
+          go mod tidy
+          make build
+
+      - name: Run Unit Tests
+        run: |
+          cd tgc
+          make test