Skip to content

Commit

Permalink
Merge branch 'cloudquery:main' into main
Browse files Browse the repository at this point in the history
  • Loading branch information
abdulrahman305 authored Sep 1, 2024
2 parents d8caa6f + 2587c31 commit 29a5ab4
Show file tree
Hide file tree
Showing 202 changed files with 28,752 additions and 1,998 deletions.
2 changes: 2 additions & 0 deletions .github/pr_labeler.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ area/plugin/destination/snowflake:
- plugins/destination/snowflake/**/*
area/plugin/destination/sqlite:
- plugins/destination/sqlite/**/*
area/plugin/destination/sqlite-python:
- plugins/destination/sqlite/**/*
area/plugin/source/airtable:
- plugins/source/airtable/**/*
area/plugin/source/bitbucket:
Expand Down
5 changes: 5 additions & 0 deletions .github/renovate.json5
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,11 @@
matchFileNames: ["plugins/destination/duckdb/**"],
enabled: false,
},
{
matchPackageNames: ["@cloudquery/plugin-config-ui-connector", "@cloudquery/plugin-config-ui-lib"],
commitMessagePrefix: "chore(deps): ",
groupName: "CloudQuery Plugin Config UI packages",
},
],
ignorePaths: [
"plugins/source/aws/**",
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/broken_links_scheduled.yml
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ jobs:
--exclude cql.ink \
--exclude egghead.io \
--exclude app.datadoghq.com \
--exclude aws.amazon.com \
${{ steps.vercel.outputs.url }}/docs \
| grep -v '───OK───' | grep -v '──SKIP──' | grep -v '0 broken'
- name: Slack Notify
Expand Down
48 changes: 48 additions & 0 deletions .github/workflows/delete_vercel_deployments.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
name: Delete Vercel Deployments from PR Comments
on:
pull_request:
types:
- closed

jobs:
delete_vercel_deployments:
runs-on: ubuntu-latest
if: ${{ !startsWith(github.event.pull_request.head.ref, 'release') }}
env:
VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }}
VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }}
VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }}
steps:
- name: Delete Vercel Deployments from PR Comments
run: |
PR_NUMBER=${{ github.event.pull_request.number }}
REPO=${{ github.repository }}
echo "Fetching comments for PR #$PR_NUMBER"
comments=$(gh api -X GET /repos/$REPO/issues/$PR_NUMBER/comments --paginate)
# Iterate through each comment
echo "$comments" | jq -c '.[]' | while read -r comment; do
body=$(echo "$comment" | jq -r '.body')
# Check if the comment contains "Unique Vercel deployment URL"
if echo "$body" | grep -q "Unique Vercel deployment URL"; then
echo "Found comment with deployment URL"
# Extract the deployment URL from the next line
deployment_url=$(echo "$body" | awk '/Unique Vercel deployment URL:/ {getline; print $1}')
if [ -n "$deployment_url" ]; then
echo "Attempting to delete deployment: $deployment_url"
# Delete the deployment using Vercel CLI
if vercel remove "$deployment_url" --token $VERCEL_TOKEN --scope $VERCEL_ORG_ID --yes; then
echo "Successfully deleted deployment: $deployment_url"
else
echo "Failed to delete deployment: $deployment_url"
fi
else
echo "No deployment URL found in the comment"
fi
fi
done
env:
GH_TOKEN: ${{ secrets.GH_CQ_BOT }}
117 changes: 116 additions & 1 deletion .github/workflows/dest_mysql.yml
Original file line number Diff line number Diff line change
Expand Up @@ -56,4 +56,119 @@ jobs:
wait-for-it -h localhost -p 3306
- name: Test MySQL
run: make test

validate-config-ui:
timeout-minutes: 15
if: github.event_name == 'pull_request'
permissions:
id-token: write
contents: read
runs-on: ubuntu-latest
defaults:
run:
working-directory: ./plugins/destination/mysql
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 2
- name: Set up Go 1.x
uses: actions/setup-go@v5
with:
go-version-file: plugins/destination/mysql/go.mod
cache: true
cache-dependency-path: plugins/destination/mysql/go.sum
- name: Build
run: go build .
- name: Use Node.js LTS
uses: actions/setup-node@v4
with:
node-version: "lts/*"
cache: "npm"
cache-dependency-path: "plugins/destination/mysql/cloud-config-ui/package-lock.json"
- name: Install dependencies
working-directory: "plugins/destination/mysql/cloud-config-ui"
run: npm ci
- name: Install dotenv
run: npm install dotenv-cli -g
- name: Install Playwright
working-directory: "plugins/destination/mysql/cloud-config-ui"
run: npx playwright install --with-deps
- name: Build config UI
working-directory: "plugins/destination/mysql/cloud-config-ui"
run: REACT_APP_USE_CLOUD_APP_MOCK=true npm run build
env:
REACT_APP_PLUGIN_TEAM: cloudquery
REACT_APP_PLUGIN_KIND: destination
REACT_APP_PLUGIN_NAME: mysql
REACT_APP_PLUGIN_VERSION: development
- name: Run E2E tests and generate configs
working-directory: "plugins/destination/mysql/cloud-config-ui"
run: E2E_TESTS_GENERATE_CONFIG=true npm run test:e2e
- name: Run unit tests
working-directory: "plugins/destination/mysql/cloud-config-ui"
run: npm run test

deploy:
if: github.event.action != 'closed' && github.event_name == 'pull_request' && !startsWith(github.head_ref, 'release')
runs-on: ubuntu-latest
env:
VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }}
VERCEL_ORG_ID: ${{ secrets.VERCEL_ORG_ID }}
VERCEL_PROJECT_ID: ${{ secrets.VERCEL_PROJECT_ID }}
defaults:
run:
working-directory: ./plugins/destination/mysql/cloud-config-ui
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-deploy
cancel-in-progress: false # to make sure that old Vercel deployments are deleted

steps:
- uses: actions/checkout@v3

- name: Install Vercel CLI
run: npm install -g vercel

- name: Deploy to Vercel
run: |
DEPLOYMENT_URL=$(vercel --token $VERCEL_TOKEN)
echo "Deployed to: $DEPLOYMENT_URL"
echo "DEPLOYMENT_URL=$DEPLOYMENT_URL" >> $GITHUB_ENV
ALIAS_DEPLOYMENT_BASE_URL=plugin-destination-mysql-cloud-ui-${{ github.event.pull_request.number }}.vercel.app
ALIAS_DEPLOYMENT_URL=https://$ALIAS_DEPLOYMENT_BASE_URL
vercel --token $VERCEL_TOKEN --scope $VERCEL_ORG_ID alias $DEPLOYMENT_URL $ALIAS_DEPLOYMENT_BASE_URL
echo "Alias deployed to: $ALIAS_DEPLOYMENT_URL"
echo "ALIAS_DEPLOYMENT_URL=$ALIAS_DEPLOYMENT_URL" >> $GITHUB_ENV
- name: Find Comment
uses: peter-evans/find-comment@v3
id: find-comment
with:
token: ${{ secrets.GH_CQ_BOT }}
issue-number: ${{ github.event.pull_request.number }}
comment-author: "cq-bot"
body-includes: "MySQL Cloud UI deployed to Vercel:"

- name: Delete Previous Vercel Deployment (if comment with URL exists)
if: steps.find-comment.outputs.comment-id != ''
run: |
echo "Comment found:"
echo "${{ steps.find-comment.outputs.comment-body }}"
PREVIOUS_DEPLOYMENT_URL=$(echo "${{ steps.find-comment.outputs.comment-body }}" | awk '/Unique Vercel deployment URL:/ {getline; print $1}')
echo "Previous deployment URL: $PREVIOUS_DEPLOYMENT_URL"
if vercel rm "$PREVIOUS_DEPLOYMENT_URL" --token $VERCEL_TOKEN --scope $VERCEL_ORG_ID --yes; then
echo "Deployment $PREVIOUS_DEPLOYMENT_URL was successfully deleted."
else
echo "Failed to delete deployment $PREVIOUS_DEPLOYMENT_URL. It may not exist or there was an error."
fi
- name: Create or update comment
uses: peter-evans/create-or-update-comment@v3
with:
token: ${{ secrets.GH_CQ_BOT }}
comment-id: ${{ steps.find-comment.outputs.comment-id }}
issue-number: ${{ github.event.pull_request.number }}
body: |
🚀 MySQL Cloud UI deployed to Vercel:
${{ env.ALIAS_DEPLOYMENT_URL }}
You can also check out this plugin in action at:
https://cloud.cloudquery.io/teams/cloudquery-test/destinations/create?plugin-cloud-ui=cloudquery|destination|mysql|${{ env.ALIAS_DEPLOYMENT_URL }}
Unique Vercel deployment URL:
${{ env.DEPLOYMENT_URL }}
edit-mode: replace
4 changes: 2 additions & 2 deletions .github/workflows/dest_postgresql.yml
Original file line number Diff line number Diff line change
Expand Up @@ -120,9 +120,9 @@ jobs:
working-directory: "plugins/destination/postgresql/cloud-config-ui"
run: E2E_TESTS_GENERATE_CONFIG=true npm run test:e2e
- name: Setup CloudQuery
uses: cloudquery/setup-cloudquery@v3
uses: cloudquery/setup-cloudquery@v4
with:
version: v6.4.0
version: v6.5.0
- name: Validate CloudQuery config
working-directory: "plugins/destination/postgresql/cloud-config-ui"
run: dotenv -e ./temp/.env -- cloudquery validate-config ./temp/config.yml
Expand Down
46 changes: 46 additions & 0 deletions .github/workflows/dest_sqlite_python.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
name: Destination Plugin SQLite-Python Workflow

concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

on:
pull_request:
paths:
- "plugins/destination/sqlite-python/**"
- ".github/workflows/dest_sqlite_python.yml"
push:
branches:
- main
paths:
- "plugins/destination/sqlite-python/**"
- ".github/workflows/dest_sqlite_python.yml"

jobs:
plugins-destination-sqlite-python:
timeout-minutes: 30
name: "plugins/destination/sqlite-python"
runs-on: ubuntu-latest
defaults:
run:
working-directory: ./plugins/destination/sqlite-python
steps:
- uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.11'
- name: Install dependencies
run: |
pip install --upgrade pip
pip install -r requirements.txt
- name: Check formatting
run: make fmt-check
-
# Required for the package command tests to work
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3

- name: Run tests
run: make test

4 changes: 2 additions & 2 deletions .github/workflows/manual_commands_triggers.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,9 @@ jobs:
token: ${{ secrets.GH_CQ_BOT }}

- name: Setup CloudQuery
uses: cloudquery/setup-cloudquery@v3
uses: cloudquery/setup-cloudquery@v4
with:
version: v6.4.0
version: v6.5.0

- name: Set up Go 1.x
uses: actions/setup-go@v5
Expand Down
28 changes: 14 additions & 14 deletions .github/workflows/publish_plugin_to_hub.yml
Original file line number Diff line number Diff line change
Expand Up @@ -117,9 +117,9 @@ jobs:

- name: Setup CloudQuery
if: needs.prepare.outputs.ui_dir_arg != ''
uses: cloudquery/setup-cloudquery@v3
uses: cloudquery/setup-cloudquery@v4
with:
version: v6.4.0
version: v6.5.0

- name: Build
working-directory: ${{ needs.prepare.outputs.plugin_dir }}
Expand All @@ -142,10 +142,10 @@ jobs:
- name: Build config UI
if: needs.prepare.outputs.ui_dir_arg != ''
env:
PLUGIN_TEAM: cloudquery
PLUGIN_KIND: ${{needs.prepare.outputs.plugin_kind}}
PLUGIN_NAME: ${{needs.prepare.outputs.plugin_name}}
PLUGIN_VERSION: ${{needs.prepare.outputs.plugin_version}}
REACT_APP_PLUGIN_TEAM: cloudquery
REACT_APP_PLUGIN_KIND: ${{needs.prepare.outputs.plugin_kind}}
REACT_APP_PLUGIN_NAME: ${{needs.prepare.outputs.plugin_name}}
REACT_APP_PLUGIN_VERSION: ${{needs.prepare.outputs.plugin_version}}
working-directory: "${{needs.prepare.outputs.plugin_dir}}/${{env.ui_dir_name}}"
run: |
npm run build
Expand Down Expand Up @@ -242,9 +242,9 @@ jobs:
java -jar app/build/libs/app.jar package -m ${{ steps.release-notes.outputs.result }} ${{ needs.prepare.outputs.plugin_version }} .
- name: Setup CloudQuery
uses: cloudquery/setup-cloudquery@v3
uses: cloudquery/setup-cloudquery@v4
with:
version: v6.4.0
version: v6.5.0

- name: Publish plugin to hub
working-directory: ${{ needs.prepare.outputs.plugin_dir }}
Expand Down Expand Up @@ -332,9 +332,9 @@ jobs:
python main.py package -m ${{ steps.release-notes.outputs.result }} ${{ needs.prepare.outputs.plugin_version }} .
- name: Setup CloudQuery
uses: cloudquery/setup-cloudquery@v3
uses: cloudquery/setup-cloudquery@v4
with:
version: v6.4.0
version: v6.5.0

- name: Publish plugin to hub
working-directory: ${{ needs.prepare.outputs.plugin_dir }}
Expand Down Expand Up @@ -416,9 +416,9 @@ jobs:
npm run dev -- package -m ${{ steps.release-notes.outputs.result }} ${{ needs.prepare.outputs.plugin_version }} .
- name: Setup CloudQuery
uses: cloudquery/setup-cloudquery@v3
uses: cloudquery/setup-cloudquery@v4
with:
version: v6.4.0
version: v6.5.0

- name: Publish plugin to hub
working-directory: ${{ needs.prepare.outputs.plugin_dir }}
Expand Down Expand Up @@ -506,9 +506,9 @@ jobs:
go run main.go package -m ${{ steps.release-notes.outputs.result }} ${{ needs.prepare.outputs.plugin_version }} .
- name: Setup CloudQuery
uses: cloudquery/setup-cloudquery@v3
uses: cloudquery/setup-cloudquery@v4
with:
version: v6.4.0
version: v6.5.0

- name: Publish plugin to hub
working-directory: ${{ needs.prepare.outputs.plugin_dir }}
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/publish_plugin_to_hub_duckdb.yml
Original file line number Diff line number Diff line change
Expand Up @@ -150,9 +150,9 @@ jobs:
run: |
go run main.go package -m ${{ steps.release-notes.outputs.result }} ${{ needs.prepare.outputs.plugin_version }} .
- name: Setup CloudQuery
uses: cloudquery/setup-cloudquery@v3
uses: cloudquery/setup-cloudquery@v4
with:
version: v6.4.0
version: v6.5.0

- name: Publish plugin to hub
working-directory: ${{ needs.prepare.outputs.plugin_dir }}
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/publish_plugin_to_hub_snowflake.yml
Original file line number Diff line number Diff line change
Expand Up @@ -151,9 +151,9 @@ jobs:
run: |
go run main.go package -m ${{ steps.release-notes.outputs.result }} ${{ needs.prepare.outputs.plugin_version }} .
- name: Setup CloudQuery
uses: cloudquery/setup-cloudquery@v3
uses: cloudquery/setup-cloudquery@v4
with:
version: v6.4.0
version: v6.5.0

- name: Publish plugin to hub
working-directory: ${{ needs.prepare.outputs.plugin_dir }}
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/publish_plugin_to_hub_sqlite.yml
Original file line number Diff line number Diff line change
Expand Up @@ -150,9 +150,9 @@ jobs:
run: |
go run main.go package -m ${{ steps.release-notes.outputs.result }} ${{ needs.prepare.outputs.plugin_version }} .
- name: Setup CloudQuery
uses: cloudquery/setup-cloudquery@v3
uses: cloudquery/setup-cloudquery@v4
with:
version: v6.4.0
version: v6.5.0

- name: Publish plugin to hub
working-directory: ${{ needs.prepare.outputs.plugin_dir }}
Expand Down
Loading

0 comments on commit 29a5ab4

Please sign in to comment.