Merge branch 'main' into issue75

.github/workflows/HighPriorityIssues.yml → .github/workflows/InternalIssuesCreateMirror.yml

@@ -1,24 +1,22 @@
-name: Create Internal issue when the "High Priority" label is applied
+name: Create or Label Mirror Issue
 on:
   issues:
     types:
       - labeled
 
 env:
   GH_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }}
-  # an event triggering this workflow is either an issue or a pull request,
-  # hence only one of the numbers will be filled in the TITLE_PREFIX
   TITLE_PREFIX: "[duckdb_mysql/#${{ github.event.issue.number }}]"
   PUBLIC_ISSUE_TITLE: ${{ github.event.issue.title }}
 
 jobs:
   create_or_label_issue:
-    if: github.event.label.name == 'High Priority'
+    if: github.event.label.name == 'reproduced' || github.event.label.name == 'under review'
     runs-on: ubuntu-latest
     steps:
       - name: Get mirror issue number
         run: |
-          gh issue list --repo duckdblabs/duckdb-internal --search "${TITLE_PREFIX}" --json title,number --jq ".[] | select(.title | startswith(\"$TITLE_PREFIX\")).number" > mirror_issue_number.txt
+          gh issue list --repo duckdblabs/duckdb-internal --search "${TITLE_PREFIX}" --json title,number --state all --jq ".[] | select(.title | startswith(\"$TITLE_PREFIX\")).number" > mirror_issue_number.txt
           echo "MIRROR_ISSUE_NUMBER=$(cat mirror_issue_number.txt)" >> $GITHUB_ENV
 
       - name: Print whether mirror issue exists
@@ -31,6 +29,7 @@ jobs:
 
       - name: Create or label issue
         run: |
+          export LABEL="mysql"
           if [ "$MIRROR_ISSUE_NUMBER" == "" ]; then
-            gh issue create --repo duckdblabs/duckdb-internal --label "extension" --label "High Priority" --title "$TITLE_PREFIX - $PUBLIC_ISSUE_TITLE" --body "See https://github.com/duckdb/duckdb_mysql/issues/${{ github.event.issue.number }}"
+            gh issue create --repo duckdblabs/duckdb-internal --label "$LABEL" --title "$TITLE_PREFIX - $PUBLIC_ISSUE_TITLE" --body "See https://github.com/duckdb/duckdb_mysql/issues/${{ github.event.issue.number }}"
           fi

.github/workflows/InternalIssuesUpdateMirror.yml

@@ -0,0 +1,48 @@
+name: Update Mirror Issue
+on:
+  issues:
+    types:
+      - closed
+      - reopened
+
+env:
+  GH_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }}
+  TITLE_PREFIX: "[duckdb_mysql/#${{ github.event.issue.number }}]"
+
+jobs:
+  update_mirror_issue:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get mirror issue number
+        run: |
+          gh issue list --repo duckdblabs/duckdb-internal --search "${TITLE_PREFIX}" --json title,number --state all --jq ".[] | select(.title | startswith(\"$TITLE_PREFIX\")).number" > mirror_issue_number.txt
+          echo "MIRROR_ISSUE_NUMBER=$(cat mirror_issue_number.txt)" >> $GITHUB_ENV
+
+      - name: Print whether mirror issue exists
+        run: |
+          if [ "$MIRROR_ISSUE_NUMBER" == "" ]; then
+            echo "Mirror issue with title prefix '$TITLE_PREFIX' does not exist yet"
+          else
+            echo "Mirror issue with title prefix '$TITLE_PREFIX' exists with number $MIRROR_ISSUE_NUMBER"
+          fi
+
+      - name: Add comment with status to mirror issue
+        run: |
+          if [ "$MIRROR_ISSUE_NUMBER" != "" ]; then
+            gh issue comment --repo duckdblabs/duckdb-internal $MIRROR_ISSUE_NUMBER --body "The issue has been ${{ github.event.action }} (https://github.com/duckdb/duckdb_mysql/issues/${{ github.event.issue.number }})."
+          fi
+
+      - name: Add closed label to mirror issue
+        if: github.event.action == 'closed'
+        run: |
+          if [ "$MIRROR_ISSUE_NUMBER" != "" ]; then
+            gh issue edit --repo duckdblabs/duckdb-internal $MIRROR_ISSUE_NUMBER --add-label "public closed" --remove-label "public reopened"
+          fi
+
+      - name: Reopen mirror issue and add reopened label
+        if: github.event.action == 'reopened'
+        run: |
+          if [ "$MIRROR_ISSUE_NUMBER" != "" ]; then
+            gh issue reopen --repo duckdblabs/duckdb-internal $MIRROR_ISSUE_NUMBER
+            gh issue edit --repo duckdblabs/duckdb-internal $MIRROR_ISSUE_NUMBER --add-label "public reopened" --remove-label "public closed"
+          fi

.github/workflows/MainDistributionPipeline.yml

@@ -14,29 +14,20 @@ concurrency:
 jobs:
   duckdb-stable-build:
     name: Build extension binaries
-    uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v0.10.1
+    uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main
     with:
       extension_name: mysql_scanner
-      duckdb_version: 'v0.10.1'
-      exclude_archs: 'osx_amd64;osx_arm64;wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools'
+      duckdb_version: main
+      exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools'
       build_duckdb_shell: false
 
-  # Note: this workaround is required for building MacOS where extra toolchain config is required
-  duckdb-stable-build-macos:
-    name: Build extension binaries
-    uses: ./.github/workflows/_extension_distribution_macos.yml
-    with:
-      duckdb_version: v0.10.1
-      extension_name: mysql_scanner
-
-
   duckdb-stable-deploy:
     name: Deploy extension binaries
-    needs: [duckdb-stable-build, duckdb-stable-build-macos]
-    uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@v0.10.1
+    needs: duckdb-stable-build
+    uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@main
     secrets: inherit
     with:
-      duckdb_version: v0.10.1
+      duckdb_version: main
       extension_name: mysql_scanner
       exclude_archs: 'wasm_mvp;wasm_eh;wasm_threads;windows_amd64_rtools'
       deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}

.github/workflows/MysqlTests.yml

@@ -54,7 +54,7 @@ jobs:
 
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.7'
+          python-version: '3.11'
 
       - name: Setup vcpkg
         uses: lukka/run-vcpkg@v11.1
@@ -70,9 +70,6 @@ jobs:
       - name: Build extension
         shell: bash
         run: |
-          # needed for MySQL cross compilation - see https://mariadb.com/kb/en/cross-compiling-mariadb/
-          echo 'SET(STACK_DIRECTION -1)' >> ${VCPKG_TOOLCHAIN_PATH}
-          echo 'SET(HAVE_IB_GCC_ATOMIC_BUILTINS 0)' >> ${VCPKG_TOOLCHAIN_PATH}
           make release
 
       - name: Test Extension

README.md

@@ -22,6 +22,7 @@ The connection string determines the parameters for how to connect to MySQL as a
 | database | Database name              | `MYSQL_DATABASE`       | NULL         |
 | port     | Port number                | `MYSQL_TCP_PORT`       | 0            |
 | socket   | Unix socket file name      | `MYSQL_UNIX_PORT`      | NULL         |
+| compress | Compress MySQL packet      | `MYSQL_COMPRESS`       | 1            |
 
 
 The tables in the file can be read as if they were normal DuckDB tables, but the underlying data is read directly from MySQL at query time.

src/include/mysql_connection.hpp

@@ -8,6 +8,7 @@
 
 #pragma once
 
+#include "duckdb/common/shared_ptr.hpp"
 #include "mysql_utils.hpp"
 #include "mysql_result.hpp"
 

src/include/mysql_result.hpp

@@ -63,7 +63,8 @@ class MySQLResult {
 	}
 	idx_t AffectedRows() {
 		if (affected_rows == idx_t(-1)) {
-			throw InternalException("MySQLResult::AffectedRows called for result that didn't affect any rows");
+			throw InternalException("MySQLResult::AffectedRows called for result "
+			                        "that didn't affect any rows");
 		}
 		return affected_rows;
 	}

src/include/mysql_scanner_extension.hpp

@@ -8,7 +8,7 @@
 
 using namespace duckdb;
 
-class MySQLScannerExtension : public Extension {
+class MysqlScannerExtension : public Extension {
 public:
 	std::string Name() override {
 		return "mysql_scanner";

src/include/storage/mysql_execute_query.hpp

@@ -46,7 +46,7 @@ class MySQLExecuteQuery : public PhysicalOperator {
 	}
 
 	string GetName() const override;
-	string ParamsToString() const override;
+	InsertionOrderPreservingMap<string> ParamsToString() const override;
 };
 
 } // namespace duckdb

src/include/storage/mysql_insert.hpp

@@ -53,7 +53,7 @@ class MySQLInsert : public PhysicalOperator {
 	}
 
 	string GetName() const override;
-	string ParamsToString() const override;
+	InsertionOrderPreservingMap<string> ParamsToString() const override;
 };
 
 } // namespace duckdb

src/include/storage/mysql_optimizer.hpp

@@ -13,7 +13,7 @@
 namespace duckdb {
 class MySQLOptimizer {
 public:
-	static void Optimize(ClientContext &context, OptimizerExtensionInfo *info, unique_ptr<LogicalOperator> &plan);
+	static void Optimize(OptimizerExtensionInput &input, unique_ptr<LogicalOperator> &plan);
 };
 
 } // namespace duckdb

src/include/storage/mysql_schema_entry.hpp

@@ -22,7 +22,7 @@ class MySQLSchemaEntry : public SchemaCatalogEntry {
 public:
 	optional_ptr<CatalogEntry> CreateTable(CatalogTransaction transaction, BoundCreateTableInfo &info) override;
 	optional_ptr<CatalogEntry> CreateFunction(CatalogTransaction transaction, CreateFunctionInfo &info) override;
-	optional_ptr<CatalogEntry> CreateIndex(ClientContext &context, CreateIndexInfo &info,
+	optional_ptr<CatalogEntry> CreateIndex(CatalogTransaction transaction, CreateIndexInfo &info,
 	                                       TableCatalogEntry &table) override;
 	optional_ptr<CatalogEntry> CreateView(CatalogTransaction transaction, CreateViewInfo &info) override;
 	optional_ptr<CatalogEntry> CreateSequence(CatalogTransaction transaction, CreateSequenceInfo &info) override;
@@ -34,7 +34,7 @@ class MySQLSchemaEntry : public SchemaCatalogEntry {
 	                                                CreatePragmaFunctionInfo &info) override;
 	optional_ptr<CatalogEntry> CreateCollation(CatalogTransaction transaction, CreateCollationInfo &info) override;
 	optional_ptr<CatalogEntry> CreateType(CatalogTransaction transaction, CreateTypeInfo &info) override;
-	void Alter(ClientContext &context, AlterInfo &info) override;
+	void Alter(CatalogTransaction transaction, AlterInfo &info) override;
 	void Scan(ClientContext &context, CatalogType type, const std::function<void(CatalogEntry &)> &callback) override;
 	void Scan(CatalogType type, const std::function<void(CatalogEntry &)> &callback) override;
 	void DropEntry(ClientContext &context, DropInfo &info) override;

src/include/storage/mysql_table_entry.hpp

@@ -44,7 +44,7 @@ class MySQLTableEntry : public TableCatalogEntry {
 
 	TableStorageInfo GetStorageInfo(ClientContext &context) override;
 
-	void BindUpdateConstraints(LogicalGet &get, LogicalProjection &proj, LogicalUpdate &update,
+	void BindUpdateConstraints(Binder &binder, LogicalGet &get, LogicalProjection &proj, LogicalUpdate &update,
 	                           ClientContext &context) override;
 };
 

src/mysql_connection.cpp

@@ -28,10 +28,9 @@ MySQLConnection &MySQLConnection::operator=(MySQLConnection &&other) noexcept {
 
 MySQLConnection MySQLConnection::Open(const string &connection_string) {
 	MySQLConnection result;
-	result.connection = make_shared<OwnedMySQLConnection>(MySQLUtils::Connect(connection_string));
+	result.connection = make_shared_ptr<OwnedMySQLConnection>(MySQLUtils::Connect(connection_string));
 	result.dsn = connection_string;
 	result.Execute("SET character_set_results = 'utf8mb4';");
-	result.Execute("SET sql_mode = 'no_backslash_escapes';");
 	return result;
 }