Merge branch 'main' of github.com:hansott/datafusion-sqlparser-rs into escape-literals

* 'main' of github.com:hansott/datafusion-sqlparser-rs:
  National strings: check if dialect supports backslash escape (apache#1672)
  Add support for Create Iceberg Table statement for Snowflake parser (apache#1664)
  Add support for Snowflake account privileges (apache#1666)
  Update rat_exclude_file.txt (apache#1670)
  Update verson to 0.54.0 and update changelog (apache#1668)
  Add support for Snowflake AT/BEFORE (apache#1667)
  Add support for qualified column names in JOIN ... USING (apache#1663)
  Add support for `IS [NOT] [form] NORMALIZED` (apache#1655)
  fix parsing of `INSERT INTO ... SELECT ... RETURNING ` (apache#1661)
  Add support for Snowflake column aliases that use SQL keywords (apache#1632)

Author: hansott

CHANGELOG.md

@@ -18,13 +18,17 @@
 -->
 
 # Changelog
-All notable changes to this project will be documented in this file.
+All notable changes to this project will be documented in one of the linked
+files. 
 
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
 Given that the parser produces a typed AST, any changes to the AST will
 technically be breaking and thus will result in a `0.(N+1)` version. 
 
 
 - Unreleased: Check https://github.com/sqlparser-rs/sqlparser-rs/commits/main for undocumented changes.
+- `0.54.0`: [changelog/0.54.0.md](changelog/0.54.0.md)
+- `0.53.0`: [changelog/0.53.0.md](changelog/0.53.0.md)
 - `0.52.0`: [changelog/0.52.0.md](changelog/0.52.0.md)
 - `0.51.0` and earlier: [changelog/0.51.0-pre.md](changelog/0.51.0-pre.md)

Cargo.toml

@@ -18,7 +18,7 @@
 [package]
 name = "sqlparser"
 description = "Extensible SQL Lexer and Parser with support for ANSI SQL:2011"
-version = "0.53.0"
+version = "0.54.0"
 authors = ["Apache DataFusion <dev@datafusion.apache.org>"]
 homepage = "https://github.com/apache/datafusion-sqlparser-rs"
 documentation = "https://docs.rs/sqlparser/"

changelog/0.54.0.md

@@ -0,0 +1,118 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# sqlparser-rs 0.54.0 Changelog
+
+This release consists of 57 commits from 24 contributors. See credits at the end of this changelog for more information.
+
+**Implemented enhancements:**
+
+- feat: support `INSERT INTO [TABLE] FUNCTION` of Clickhouse [#1633](https://github.com/apache/datafusion-sqlparser-rs/pull/1633) (byte-sourcerer)
+
+**Other:**
+
+- Run cargo fmt on `derive` crate [#1595](https://github.com/apache/datafusion-sqlparser-rs/pull/1595) (alamb)
+- Add Apache license header to spans.rs [#1594](https://github.com/apache/datafusion-sqlparser-rs/pull/1594) (alamb)
+- Add support for BigQuery `ANY TYPE` data type [#1602](https://github.com/apache/datafusion-sqlparser-rs/pull/1602) (MartinSahlen)
+- Add support for TABLESAMPLE [#1580](https://github.com/apache/datafusion-sqlparser-rs/pull/1580) (yoavcloud)
+- Redshift: Fix parsing for quoted numbered columns [#1576](https://github.com/apache/datafusion-sqlparser-rs/pull/1576) (7phs)
+- Add the alter table ON COMMIT option to Snowflake [#1606](https://github.com/apache/datafusion-sqlparser-rs/pull/1606) (yoavcloud)
+- Support parsing `EXPLAIN ESTIMATE` of Clickhouse [#1605](https://github.com/apache/datafusion-sqlparser-rs/pull/1605) (byte-sourcerer)
+- Fix BigQuery hyphenated ObjectName with numbers [#1598](https://github.com/apache/datafusion-sqlparser-rs/pull/1598) (ayman-sigma)
+- Fix test compilation issue [#1609](https://github.com/apache/datafusion-sqlparser-rs/pull/1609) (iffyio)
+- Allow foreign table constraint without columns [#1608](https://github.com/apache/datafusion-sqlparser-rs/pull/1608) (ramnivas)
+- Support optional table for `ANALYZE` statement [#1599](https://github.com/apache/datafusion-sqlparser-rs/pull/1599) (yuyang-ok)
+- Support DOUBLE data types with precision for Mysql [#1611](https://github.com/apache/datafusion-sqlparser-rs/pull/1611) (artorias1024)
+- Add `#[recursive]` [#1522](https://github.com/apache/datafusion-sqlparser-rs/pull/1522) (blaginin)
+- Support arbitrary composite access expressions [#1600](https://github.com/apache/datafusion-sqlparser-rs/pull/1600) (ayman-sigma)
+- Consolidate `MapAccess`, and `Subscript` into `CompoundExpr` to handle the complex field access chain  [#1551](https://github.com/apache/datafusion-sqlparser-rs/pull/1551) (goldmedal)
+- Handle empty projection in Postgres SELECT  statements [#1613](https://github.com/apache/datafusion-sqlparser-rs/pull/1613) (tobyhede)
+- Merge composite and compound expr test cases [#1615](https://github.com/apache/datafusion-sqlparser-rs/pull/1615) (iffyio)
+- Support Snowflake Update-From-Select [#1604](https://github.com/apache/datafusion-sqlparser-rs/pull/1604) (yuval-illumex)
+- Improve parsing performance by reducing token cloning [#1587](https://github.com/apache/datafusion-sqlparser-rs/pull/1587) (davisp)
+- Improve Parser documentation [#1617](https://github.com/apache/datafusion-sqlparser-rs/pull/1617) (alamb)
+- Add support for DROP EXTENSION [#1610](https://github.com/apache/datafusion-sqlparser-rs/pull/1610) (ramnivas)
+- Refactor advancing token to avoid duplication, avoid borrow checker issues [#1618](https://github.com/apache/datafusion-sqlparser-rs/pull/1618) (alamb)
+- Fix the parsing result for the special double number [#1621](https://github.com/apache/datafusion-sqlparser-rs/pull/1621) (goldmedal)
+- SQLite: Allow dollar signs in placeholder names [#1620](https://github.com/apache/datafusion-sqlparser-rs/pull/1620) (hansott)
+- Improve error for an unexpected token after DROP [#1623](https://github.com/apache/datafusion-sqlparser-rs/pull/1623) (ramnivas)
+- Fix `sqlparser_bench` benchmark compilation [#1625](https://github.com/apache/datafusion-sqlparser-rs/pull/1625) (alamb)
+- Improve parsing speed by avoiding some clones in parse_identifier [#1624](https://github.com/apache/datafusion-sqlparser-rs/pull/1624) (alamb)
+- Simplify `parse_keyword_apis` more [#1626](https://github.com/apache/datafusion-sqlparser-rs/pull/1626) (alamb)
+- Test benchmarks and Improve benchmark README.md [#1627](https://github.com/apache/datafusion-sqlparser-rs/pull/1627) (alamb)
+- Add support for MYSQL's `RENAME TABLE` [#1616](https://github.com/apache/datafusion-sqlparser-rs/pull/1616) (wugeer)
+- Correctly tokenize nested comments [#1629](https://github.com/apache/datafusion-sqlparser-rs/pull/1629) (hansott)
+- Add support for USE SECONDARY ROLE (vs. ROLES) [#1637](https://github.com/apache/datafusion-sqlparser-rs/pull/1637) (yoavcloud)
+- Add support for various Snowflake grantees [#1640](https://github.com/apache/datafusion-sqlparser-rs/pull/1640) (yoavcloud)
+- Add support for the SQL OVERLAPS predicate [#1638](https://github.com/apache/datafusion-sqlparser-rs/pull/1638) (yoavcloud)
+- Add support for Snowflake LIST and REMOVE [#1639](https://github.com/apache/datafusion-sqlparser-rs/pull/1639) (yoavcloud)
+- Add support for MySQL's INSERT INTO ... SET syntax [#1641](https://github.com/apache/datafusion-sqlparser-rs/pull/1641) (yoavcloud)
+- Start new line if \r in Postgres dialect [#1647](https://github.com/apache/datafusion-sqlparser-rs/pull/1647) (hansott)
+- Support pluralized time units [#1630](https://github.com/apache/datafusion-sqlparser-rs/pull/1630) (wugeer)
+- Replace `ReferentialAction` enum in `DROP` statements [#1648](https://github.com/apache/datafusion-sqlparser-rs/pull/1648) (stepancheg)
+- Add support for MS-SQL BEGIN/END TRY/CATCH [#1649](https://github.com/apache/datafusion-sqlparser-rs/pull/1649) (yoavcloud)
+- Fix MySQL parsing of GRANT, REVOKE, and CREATE VIEW [#1538](https://github.com/apache/datafusion-sqlparser-rs/pull/1538) (mvzink)
+- Add support for the Snowflake MINUS set operator [#1652](https://github.com/apache/datafusion-sqlparser-rs/pull/1652) (yoavcloud)
+- ALTER TABLE DROP {COLUMN|CONSTRAINT} RESTRICT [#1651](https://github.com/apache/datafusion-sqlparser-rs/pull/1651) (stepancheg)
+- Add support for ClickHouse `FORMAT` on `INSERT` [#1628](https://github.com/apache/datafusion-sqlparser-rs/pull/1628) (bombsimon)
+- MsSQL SET for session params [#1646](https://github.com/apache/datafusion-sqlparser-rs/pull/1646) (yoavcloud)
+- Correctly look for end delimiter dollar quoted string [#1650](https://github.com/apache/datafusion-sqlparser-rs/pull/1650) (hansott)
+- Support single line comments starting with '#' for Hive [#1654](https://github.com/apache/datafusion-sqlparser-rs/pull/1654) (wugeer)
+- Support trailing commas in `FROM` clause [#1645](https://github.com/apache/datafusion-sqlparser-rs/pull/1645) (barsela1)
+- Allow empty options for BigQuery [#1657](https://github.com/apache/datafusion-sqlparser-rs/pull/1657) (MartinSahlen)
+- Add support for parsing RAISERROR [#1656](https://github.com/apache/datafusion-sqlparser-rs/pull/1656) (AvivDavid-Satori)
+- Add support for Snowflake column aliases that use SQL keywords [#1632](https://github.com/apache/datafusion-sqlparser-rs/pull/1632) (yoavcloud)
+- fix parsing of `INSERT INTO ... SELECT ... RETURNING ` [#1661](https://github.com/apache/datafusion-sqlparser-rs/pull/1661) (lovasoa)
+- Add support for `IS [NOT] [form] NORMALIZED` [#1655](https://github.com/apache/datafusion-sqlparser-rs/pull/1655) (alexander-beedie)
+- Add support for qualified column names in JOIN ... USING [#1663](https://github.com/apache/datafusion-sqlparser-rs/pull/1663) (yoavcloud)
+- Add support for Snowflake AT/BEFORE [#1667](https://github.com/apache/datafusion-sqlparser-rs/pull/1667) (yoavcloud)
+
+## Credits
+
+Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor.
+
+```
+    13	Yoav Cohen
+     9	Andrew Lamb
+     4	Hans Ott
+     3	Ramnivas Laddad
+     3	wugeer
+     2	Ayman Elkfrawy
+     2	Ifeanyi Ubah
+     2	Jax Liu
+     2	Martin Abelson Sahlen
+     2	Stepan Koltsov
+     2	cjw
+     1	Aleksei Piianin
+     1	Alexander Beedie
+     1	AvivDavid-Satori
+     1	Dmitrii Blaginin
+     1	Michael Victor Zink
+     1	Ophir LOJKINE
+     1	Paul J. Davis
+     1	Simon Sawert
+     1	Toby Hede
+     1	Yuval Shkolar
+     1	artorias1024
+     1	bar sela
+     1	yuyang
+```
+
+Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release.
+

dev/release/rat_exclude_files.txt

@@ -3,4 +3,5 @@
 .tool-versions
 dev/release/rat_exclude_files.txt
 fuzz/.gitignore
+sqlparser_bench/img/flamegraph.svg
 

src/ast/dml.rs

@@ -36,7 +36,8 @@ use super::{
     CommentDef, Expr, FileFormat, FromTable, HiveDistributionStyle, HiveFormat, HiveIOFormat,
     HiveRowFormat, Ident, InsertAliases, MysqlInsertPriority, ObjectName, OnCommit, OnInsert,
     OneOrManyWithParens, OrderByExpr, Query, RowAccessPolicy, SelectItem, Setting, SqlOption,
-    SqliteOnConflict, TableEngine, TableObject, TableWithJoins, Tag, WrappedCollection,
+    SqliteOnConflict, StorageSerializationPolicy, TableEngine, TableObject, TableWithJoins, Tag,
+    WrappedCollection,
 };
 
 /// CREATE INDEX statement.
@@ -117,6 +118,7 @@ pub struct CreateTable {
     pub if_not_exists: bool,
     pub transient: bool,
     pub volatile: bool,
+    pub iceberg: bool,
     /// Table name
     #[cfg_attr(feature = "visitor", visit(with = "visit_relation"))]
     pub name: ObjectName,
@@ -192,6 +194,21 @@ pub struct CreateTable {
     /// Snowflake "WITH TAG" clause
     /// <https://docs.snowflake.com/en/sql-reference/sql/create-table>
     pub with_tags: Option<Vec<Tag>>,
+    /// Snowflake "EXTERNAL_VOLUME" clause for Iceberg tables
+    /// <https://docs.snowflake.com/en/sql-reference/sql/create-iceberg-table>
+    pub external_volume: Option<String>,
+    /// Snowflake "BASE_LOCATION" clause for Iceberg tables
+    /// <https://docs.snowflake.com/en/sql-reference/sql/create-iceberg-table>
+    pub base_location: Option<String>,
+    /// Snowflake "CATALOG" clause for Iceberg tables
+    /// <https://docs.snowflake.com/en/sql-reference/sql/create-iceberg-table>
+    pub catalog: Option<String>,
+    /// Snowflake "CATALOG_SYNC" clause for Iceberg tables
+    /// <https://docs.snowflake.com/en/sql-reference/sql/create-iceberg-table>
+    pub catalog_sync: Option<String>,
+    /// Snowflake "STORAGE_SERIALIZATION_POLICY" clause for Iceberg tables
+    /// <https://docs.snowflake.com/en/sql-reference/sql/create-iceberg-table>
+    pub storage_serialization_policy: Option<StorageSerializationPolicy>,
 }
 
 impl Display for CreateTable {
@@ -205,7 +222,7 @@ impl Display for CreateTable {
         //   `CREATE TABLE t (a INT) AS SELECT a from t2`
         write!(
             f,
-            "CREATE {or_replace}{external}{global}{temporary}{transient}{volatile}TABLE {if_not_exists}{name}",
+            "CREATE {or_replace}{external}{global}{temporary}{transient}{volatile}{iceberg}TABLE {if_not_exists}{name}",
             or_replace = if self.or_replace { "OR REPLACE " } else { "" },
             external = if self.external { "EXTERNAL " } else { "" },
             global = self.global
@@ -221,6 +238,8 @@ impl Display for CreateTable {
             temporary = if self.temporary { "TEMPORARY " } else { "" },
             transient = if self.transient { "TRANSIENT " } else { "" },
             volatile = if self.volatile { "VOLATILE " } else { "" },
+            // Only for Snowflake
+            iceberg = if self.iceberg { "ICEBERG " } else { "" },
             name = self.name,
         )?;
         if let Some(on_cluster) = &self.on_cluster {
@@ -382,6 +401,31 @@ impl Display for CreateTable {
             )?;
         }
 
+        if let Some(external_volume) = self.external_volume.as_ref() {
+            write!(f, " EXTERNAL_VOLUME = '{external_volume}'")?;
+        }
+
+        if let Some(catalog) = self.catalog.as_ref() {
+            write!(f, " CATALOG = '{catalog}'")?;
+        }
+
+        if self.iceberg {
+            if let Some(base_location) = self.base_location.as_ref() {
+                write!(f, " BASE_LOCATION = '{base_location}'")?;
+            }
+        }
+
+        if let Some(catalog_sync) = self.catalog_sync.as_ref() {
+            write!(f, " CATALOG_SYNC = '{catalog_sync}'")?;
+        }
+
+        if let Some(storage_serialization_policy) = self.storage_serialization_policy.as_ref() {
+            write!(
+                f,
+                " STORAGE_SERIALIZATION_POLICY = {storage_serialization_policy}"
+            )?;
+        }
+
         if self.copy_grants {
             write!(f, " COPY GRANTS")?;
         }

src/ast/helpers/stmt_create_table.rs

@@ -28,7 +28,7 @@ use super::super::dml::CreateTable;
 use crate::ast::{
     ClusteredBy, ColumnDef, CommentDef, Expr, FileFormat, HiveDistributionStyle, HiveFormat, Ident,
     ObjectName, OnCommit, OneOrManyWithParens, Query, RowAccessPolicy, SqlOption, Statement,
-    TableConstraint, TableEngine, Tag, WrappedCollection,
+    StorageSerializationPolicy, TableConstraint, TableEngine, Tag, WrappedCollection,
 };
 use crate::parser::ParserError;
 
@@ -71,6 +71,7 @@ pub struct CreateTableBuilder {
     pub if_not_exists: bool,
     pub transient: bool,
     pub volatile: bool,
+    pub iceberg: bool,
     pub name: ObjectName,
     pub columns: Vec<ColumnDef>,
     pub constraints: Vec<TableConstraint>,
@@ -107,6 +108,11 @@ pub struct CreateTableBuilder {
     pub with_aggregation_policy: Option<ObjectName>,
     pub with_row_access_policy: Option<RowAccessPolicy>,
     pub with_tags: Option<Vec<Tag>>,
+    pub base_location: Option<String>,
+    pub external_volume: Option<String>,
+    pub catalog: Option<String>,
+    pub catalog_sync: Option<String>,
+    pub storage_serialization_policy: Option<StorageSerializationPolicy>,
 }
 
 impl CreateTableBuilder {
@@ -119,6 +125,7 @@ impl CreateTableBuilder {
             if_not_exists: false,
             transient: false,
             volatile: false,
+            iceberg: false,
             name,
             columns: vec![],
             constraints: vec![],
@@ -155,6 +162,11 @@ impl CreateTableBuilder {
             with_aggregation_policy: None,
             with_row_access_policy: None,
             with_tags: None,
+            base_location: None,
+            external_volume: None,
+            catalog: None,
+            catalog_sync: None,
+            storage_serialization_policy: None,
         }
     }
     pub fn or_replace(mut self, or_replace: bool) -> Self {
@@ -192,6 +204,11 @@ impl CreateTableBuilder {
         self
     }
 
+    pub fn iceberg(mut self, iceberg: bool) -> Self {
+        self.iceberg = iceberg;
+        self
+    }
+
     pub fn columns(mut self, columns: Vec<ColumnDef>) -> Self {
         self.columns = columns;
         self
@@ -371,6 +388,34 @@ impl CreateTableBuilder {
         self
     }
 
+    pub fn base_location(mut self, base_location: Option<String>) -> Self {
+        self.base_location = base_location;
+        self
+    }
+
+    pub fn external_volume(mut self, external_volume: Option<String>) -> Self {
+        self.external_volume = external_volume;
+        self
+    }
+
+    pub fn catalog(mut self, catalog: Option<String>) -> Self {
+        self.catalog = catalog;
+        self
+    }
+
+    pub fn catalog_sync(mut self, catalog_sync: Option<String>) -> Self {
+        self.catalog_sync = catalog_sync;
+        self
+    }
+
+    pub fn storage_serialization_policy(
+        mut self,
+        storage_serialization_policy: Option<StorageSerializationPolicy>,
+    ) -> Self {
+        self.storage_serialization_policy = storage_serialization_policy;
+        self
+    }
+
     pub fn build(self) -> Statement {
         Statement::CreateTable(CreateTable {
             or_replace: self.or_replace,
@@ -380,6 +425,7 @@ impl CreateTableBuilder {
             if_not_exists: self.if_not_exists,
             transient: self.transient,
             volatile: self.volatile,
+            iceberg: self.iceberg,
             name: self.name,
             columns: self.columns,
             constraints: self.constraints,
@@ -416,6 +462,11 @@ impl CreateTableBuilder {
             with_aggregation_policy: self.with_aggregation_policy,
             with_row_access_policy: self.with_row_access_policy,
             with_tags: self.with_tags,
+            base_location: self.base_location,
+            external_volume: self.external_volume,
+            catalog: self.catalog,
+            catalog_sync: self.catalog_sync,
+            storage_serialization_policy: self.storage_serialization_policy,
         })
     }
 }
@@ -435,6 +486,7 @@ impl TryFrom<Statement> for CreateTableBuilder {
                 if_not_exists,
                 transient,
                 volatile,
+                iceberg,
                 name,
                 columns,
                 constraints,
@@ -471,6 +523,11 @@ impl TryFrom<Statement> for CreateTableBuilder {
                 with_aggregation_policy,
                 with_row_access_policy,
                 with_tags,
+                base_location,
+                external_volume,
+                catalog,
+                catalog_sync,
+                storage_serialization_policy,
             }) => Ok(Self {
                 or_replace,
                 temporary,
@@ -505,6 +562,7 @@ impl TryFrom<Statement> for CreateTableBuilder {
                 clustered_by,
                 options,
                 strict,
+                iceberg,
                 copy_grants,
                 enable_schema_evolution,
                 change_tracking,
@@ -515,6 +573,11 @@ impl TryFrom<Statement> for CreateTableBuilder {
                 with_row_access_policy,
                 with_tags,
                 volatile,
+                base_location,
+                external_volume,
+                catalog,
+                catalog_sync,
+                storage_serialization_policy,
             }),
             _ => Err(ParserError::ParserError(format!(
                 "Expected create table statement, but received: {stmt}"