diff --git a/backends-clickhouse/pom.xml b/backends-clickhouse/pom.xml index 6bc9f8ec5336..6314575bf47b 100644 --- a/backends-clickhouse/pom.xml +++ b/backends-clickhouse/pom.xml @@ -269,7 +269,7 @@ com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier} - src/main/resources/org/apache/spark/sql/execution/datasources/v1 + src/main/resources/org/apache/spark/sql/execution/datasources/ch false diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala index 773cd35e9367..3c92b6112e89 100644 --- a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala +++ b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala @@ -30,8 +30,7 @@ import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.execution.{SparkPlan, SQLExecution} import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec import org.apache.spark.sql.execution.datasources.{BasicWriteJobStatsTracker, FakeRowAdaptor, FileFormatWriter, WriteJobStatsTracker} -import org.apache.spark.sql.execution.datasources.v1.clickhouse.MergeTreeFileFormatWriter -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig +import org.apache.spark.sql.execution.datasources.mergetree.{ClickHouseConfig, MergeTreeFileFormatWriter} import org.apache.spark.util.{Clock, SerializableConfiguration} import org.apache.commons.lang3.exception.ExceptionUtils diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/DeltaLog.scala b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/DeltaLog.scala index 57c6c8550fc0..85d94ca0df74 100644 --- a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/DeltaLog.scala +++ b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/DeltaLog.scala @@ -31,7 +31,7 @@ import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.schema.{SchemaMergingUtils, SchemaUtils} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.storage.LogStoreProvider -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig +import org.apache.spark.sql.execution.datasources.mergetree.ClickHouseConfig import org.apache.spark.sql.execution.datasources._ import org.apache.spark.sql.sources.{BaseRelation, InsertableRelation} import org.apache.spark.sql.types.{StructField, StructType} diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/Snapshot.scala b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/Snapshot.scala index 2e4d6bb2207c..94efacd091e2 100644 --- a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/Snapshot.scala +++ b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/Snapshot.scala @@ -27,7 +27,7 @@ import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.stats.{DataSkippingReader, DeltaScan, FileSizeHistogram, StatisticsCollection} import org.apache.spark.sql.delta.util.StateCache import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation} -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig +import org.apache.spark.sql.execution.datasources.mergetree.ClickHouseConfig import org.apache.spark.sql.functions._ import org.apache.spark.sql.types.StructType import org.apache.spark.util.{SerializableConfiguration, Utils} diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/catalog/ClickHouseSparkCatalog.scala similarity index 98% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala rename to backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/catalog/ClickHouseSparkCatalog.scala index 61e1da44d0af..54ed7798a63f 100644 --- a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala +++ b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/catalog/ClickHouseSparkCatalog.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.execution.datasources.v2.clickhouse +package org.apache.spark.sql.delta.catalog import org.apache.gluten.sql.shims.SparkShimLoader @@ -29,12 +29,12 @@ import org.apache.spark.sql.connector.expressions.Transform import org.apache.spark.sql.connector.write.{LogicalWriteInfo, V1Write, WriteBuilder} import org.apache.spark.sql.delta.{DeltaConfigs, DeltaErrors, DeltaLog, DeltaOptions, DeltaTableUtils} import org.apache.spark.sql.delta.DeltaTableIdentifier.gluePermissionError -import org.apache.spark.sql.delta.catalog.{ClickHouseTableV2, DeltaTableV2, TempClickHouseTableV2} import org.apache.spark.sql.delta.commands.{CreateDeltaTableCommand, TableCreationModes, WriteIntoDelta} import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.{DeltaSourceUtils, DeltaSQLConf} import org.apache.spark.sql.execution.datasources.{DataSource, PartitioningUtils} -import org.apache.spark.sql.execution.datasources.v2.clickhouse.utils.CHDataSourceUtils +import org.apache.spark.sql.execution.datasources.mergetree.ClickHouseConfig +import org.apache.spark.sql.execution.datasources.mergetree.utils.CHDataSourceUtils import org.apache.spark.sql.sources.InsertableRelation import org.apache.spark.sql.types.StructType diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala index 25f691d2e8cb..2722856cf9dd 100644 --- a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala +++ b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala @@ -26,9 +26,8 @@ import org.apache.spark.sql.delta.actions.Metadata import org.apache.spark.sql.delta.catalog.ClickHouseTableV2.deltaLog2Table import org.apache.spark.sql.delta.sources.DeltaDataSource import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, PartitionDirectory} -import org.apache.spark.sql.execution.datasources.clickhouse.utils.MergeTreePartsPartitionsUtil -import org.apache.spark.sql.execution.datasources.mergetree.StorageMeta -import org.apache.spark.sql.execution.datasources.v2.clickhouse.source.DeltaMergeTreeFileFormat +import org.apache.spark.sql.execution.datasources.mergetree.{DeltaMergeTreeFileFormat, StorageMeta} +import org.apache.spark.sql.execution.datasources.mergetree.utils.MergeTreePartsPartitionsUtil import org.apache.spark.sql.util.CaseInsensitiveStringMap import org.apache.spark.util.collection.BitSet diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala index 6437ab75903b..09e19c5851f7 100644 --- a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala +++ b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala @@ -32,9 +32,9 @@ import org.apache.spark.sql.delta.schema.SchemaUtils import org.apache.spark.sql.delta.skipping.MultiDimClustering import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.execution.command.{LeafRunnableCommand, RunnableCommand} -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig -import org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata.AddMergeTreeParts -import org.apache.spark.sql.execution.datasources.v2.clickhouse.utils.CHDataSourceUtils +import org.apache.spark.sql.execution.datasources.mergetree.ClickHouseConfig +import org.apache.spark.sql.execution.datasources.mergetree.metadata.AddMergeTreeParts +import org.apache.spark.sql.execution.datasources.mergetree.utils.CHDataSourceUtils import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.execution.metric.SQLMetrics.createMetric import org.apache.spark.sql.types._ diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala index 242a5c3c277a..ad2293976b33 100644 --- a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala +++ b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala @@ -31,10 +31,10 @@ import org.apache.spark.sql.delta.actions.{AddFile, FileAction} import org.apache.spark.sql.delta.catalog.ClickHouseTableV2 import org.apache.spark.sql.errors.QueryExecutionErrors import org.apache.spark.sql.execution.datasources.CHDatasourceJniWrapper -import org.apache.spark.sql.execution.datasources.v1.CHMergeTreeWriterInjects -import org.apache.spark.sql.execution.datasources.v1.clickhouse._ -import org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata.{AddFileTags, AddMergeTreeParts} -import org.apache.spark.sql.execution.datasources.v2.clickhouse.utils.CHDataSourceUtils +import org.apache.spark.sql.execution.datasources.ch.CHMergeTreeWriterInjects +import org.apache.spark.sql.execution.datasources.mergetree.MergeTreeWriteTaskResult +import org.apache.spark.sql.execution.datasources.mergetree.metadata.{AddFileTags, AddMergeTreeParts} +import org.apache.spark.sql.execution.datasources.mergetree.utils.CHDataSourceUtils import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.StructType import org.apache.spark.util.{SerializableConfiguration, SystemClock, Utils} diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/VacuumCommand.scala b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/VacuumCommand.scala index c5527933b2fc..bb52c5e78477 100644 --- a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/VacuumCommand.scala +++ b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/delta/commands/VacuumCommand.scala @@ -23,7 +23,7 @@ import org.apache.spark.sql.delta._ import org.apache.spark.sql.delta.actions.{FileAction, RemoveFile} import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.delta.util.DeltaFileOperations -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig +import org.apache.spark.sql.execution.datasources.mergetree.ClickHouseConfig import org.apache.spark.sql.functions.{col, expr, when} import org.apache.spark.util.{Clock, SerializableConfiguration, SystemClock} diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/datasources/mergetree/ClickHouseDataSource.scala similarity index 98% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala rename to backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/datasources/mergetree/ClickHouseDataSource.scala index 8c1062f4c7b6..96a018579f4e 100644 --- a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala +++ b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/datasources/mergetree/ClickHouseDataSource.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.execution.datasources.v2.clickhouse +package org.apache.spark.sql.execution.datasources.mergetree import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap @@ -133,13 +133,13 @@ class ClickHouseDataSource extends DeltaDataSource { } else { Map.empty } - (new ClickHouseTableV2( + new ClickHouseTableV2( sqlContext.sparkSession, new Path(maybePath), timeTravelOpt = timeTravelByParams, options = dfOptions, cdcOptions = new CaseInsensitiveStringMap(cdcOptions.asJava) - )).toBaseRelation + ).toBaseRelation } } } diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/datasources/mergetree/DeltaMergeTreeFileFormat.scala similarity index 92% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala rename to backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/datasources/mergetree/DeltaMergeTreeFileFormat.scala index 5b2dc164b56a..0184c976282c 100644 --- a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala +++ b/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/datasources/mergetree/DeltaMergeTreeFileFormat.scala @@ -14,14 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.execution.datasources.v2.clickhouse.source +package org.apache.spark.sql.execution.datasources.mergetree import org.apache.spark.sql.SparkSession import org.apache.spark.sql.delta.DeltaParquetFileFormat import org.apache.spark.sql.delta.actions.Metadata import org.apache.spark.sql.execution.datasources.{OutputWriter, OutputWriterFactory} -import org.apache.spark.sql.execution.datasources.mergetree.DeltaMetaReader -import org.apache.spark.sql.execution.datasources.v1.GlutenMergeTreeWriterInjects +import org.apache.spark.sql.execution.datasources.ch.GlutenMergeTreeWriterInjects import org.apache.spark.sql.types.StructType import org.apache.hadoop.mapreduce.{Job, TaskAttemptContext} diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala index 773cd35e9367..3c92b6112e89 100644 --- a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala +++ b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala @@ -30,8 +30,7 @@ import org.apache.spark.sql.delta.sources.DeltaSQLConf import org.apache.spark.sql.execution.{SparkPlan, SQLExecution} import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec import org.apache.spark.sql.execution.datasources.{BasicWriteJobStatsTracker, FakeRowAdaptor, FileFormatWriter, WriteJobStatsTracker} -import org.apache.spark.sql.execution.datasources.v1.clickhouse.MergeTreeFileFormatWriter -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig +import org.apache.spark.sql.execution.datasources.mergetree.{ClickHouseConfig, MergeTreeFileFormatWriter} import org.apache.spark.util.{Clock, SerializableConfiguration} import org.apache.commons.lang3.exception.ExceptionUtils diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/DeltaLog.scala b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/DeltaLog.scala index 78fbc3fcdb99..4f5b152de627 100644 --- a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/DeltaLog.scala +++ b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/DeltaLog.scala @@ -51,7 +51,7 @@ import org.apache.spark.sql.catalyst.expressions.{And, Attribute, Cast, Expressi import org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper import org.apache.spark.sql.catalyst.util.FailFastMode import org.apache.spark.sql.execution.datasources._ -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig +import org.apache.spark.sql.execution.datasources.mergetree.ClickHouseConfig import org.apache.spark.sql.expressions.UserDefinedFunction import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.sources.{BaseRelation, InsertableRelation} diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/Snapshot.scala b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/Snapshot.scala index b2b5ba42bb30..5353ee8922df 100644 --- a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/Snapshot.scala +++ b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/Snapshot.scala @@ -34,7 +34,7 @@ import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.expressions.Expression -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig +import org.apache.spark.sql.execution.datasources.mergetree.ClickHouseConfig import org.apache.spark.sql.functions._ import org.apache.spark.sql.types.StructType import org.apache.spark.util.Utils diff --git a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/catalog/ClickHouseSparkCatalog.scala similarity index 98% rename from backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala rename to backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/catalog/ClickHouseSparkCatalog.scala index 61e1da44d0af..b2043f2c1680 100644 --- a/backends-clickhouse/src/main/delta-20/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala +++ b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/catalog/ClickHouseSparkCatalog.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.execution.datasources.v2.clickhouse +package org.apache.spark.sql.delta.catalog import org.apache.gluten.sql.shims.SparkShimLoader @@ -29,12 +29,12 @@ import org.apache.spark.sql.connector.expressions.Transform import org.apache.spark.sql.connector.write.{LogicalWriteInfo, V1Write, WriteBuilder} import org.apache.spark.sql.delta.{DeltaConfigs, DeltaErrors, DeltaLog, DeltaOptions, DeltaTableUtils} import org.apache.spark.sql.delta.DeltaTableIdentifier.gluePermissionError -import org.apache.spark.sql.delta.catalog.{ClickHouseTableV2, DeltaTableV2, TempClickHouseTableV2} import org.apache.spark.sql.delta.commands.{CreateDeltaTableCommand, TableCreationModes, WriteIntoDelta} import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.sources.{DeltaSourceUtils, DeltaSQLConf} import org.apache.spark.sql.execution.datasources.{DataSource, PartitioningUtils} -import org.apache.spark.sql.execution.datasources.v2.clickhouse.utils.CHDataSourceUtils +import org.apache.spark.sql.execution.datasources.mergetree.ClickHouseConfig +import org.apache.spark.sql.execution.datasources.mergetree.utils.CHDataSourceUtils import org.apache.spark.sql.sources.InsertableRelation import org.apache.spark.sql.types.StructType @@ -607,7 +607,7 @@ class ClickHouseSparkCatalog override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = table match { case supportsWrite: SupportsWrite => supportsWrite.newWriteBuilder(info) - case _ => throw DeltaErrors.unsupportedWriteStagedTable(name) + case _ => throw DeltaErrors.unsupportedWriteStagedTable(name()) } } } diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala index 25f691d2e8cb..2722856cf9dd 100644 --- a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala +++ b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala @@ -26,9 +26,8 @@ import org.apache.spark.sql.delta.actions.Metadata import org.apache.spark.sql.delta.catalog.ClickHouseTableV2.deltaLog2Table import org.apache.spark.sql.delta.sources.DeltaDataSource import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, PartitionDirectory} -import org.apache.spark.sql.execution.datasources.clickhouse.utils.MergeTreePartsPartitionsUtil -import org.apache.spark.sql.execution.datasources.mergetree.StorageMeta -import org.apache.spark.sql.execution.datasources.v2.clickhouse.source.DeltaMergeTreeFileFormat +import org.apache.spark.sql.execution.datasources.mergetree.{DeltaMergeTreeFileFormat, StorageMeta} +import org.apache.spark.sql.execution.datasources.mergetree.utils.MergeTreePartsPartitionsUtil import org.apache.spark.sql.util.CaseInsensitiveStringMap import org.apache.spark.util.collection.BitSet diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala index 7fa2c97d9006..ed1e9396625e 100644 --- a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala +++ b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala @@ -38,9 +38,9 @@ import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Expression} import org.apache.spark.sql.execution.command.{LeafRunnableCommand, RunnableCommand} -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig -import org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata.AddMergeTreeParts -import org.apache.spark.sql.execution.datasources.v2.clickhouse.utils.CHDataSourceUtils +import org.apache.spark.sql.execution.datasources.mergetree.ClickHouseConfig +import org.apache.spark.sql.execution.datasources.mergetree.metadata.AddMergeTreeParts +import org.apache.spark.sql.execution.datasources.mergetree.utils.CHDataSourceUtils import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.execution.metric.SQLMetrics.createMetric import org.apache.spark.sql.types._ diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala index e7b8ce1d3053..e05f76f871d2 100644 --- a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala +++ b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala @@ -31,10 +31,10 @@ import org.apache.spark.sql.delta.actions.{AddFile, FileAction} import org.apache.spark.sql.delta.catalog.ClickHouseTableV2 import org.apache.spark.sql.errors.QueryExecutionErrors import org.apache.spark.sql.execution.datasources.CHDatasourceJniWrapper -import org.apache.spark.sql.execution.datasources.v1.CHMergeTreeWriterInjects -import org.apache.spark.sql.execution.datasources.v1.clickhouse._ -import org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata.{AddFileTags, AddMergeTreeParts} -import org.apache.spark.sql.execution.datasources.v2.clickhouse.utils.CHDataSourceUtils +import org.apache.spark.sql.execution.datasources.ch.CHMergeTreeWriterInjects +import org.apache.spark.sql.execution.datasources.mergetree.MergeTreeWriteTaskResult +import org.apache.spark.sql.execution.datasources.mergetree.metadata.{AddFileTags, AddMergeTreeParts} +import org.apache.spark.sql.execution.datasources.mergetree.utils.CHDataSourceUtils import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.StructType import org.apache.spark.util.{SerializableConfiguration, SystemClock, Utils} diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/VacuumCommand.scala b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/VacuumCommand.scala index 9f82feeee2fc..199914719423 100644 --- a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/VacuumCommand.scala +++ b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/delta/commands/VacuumCommand.scala @@ -36,7 +36,7 @@ import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.gluten.utils.QueryPlanSelector import org.apache.spark.broadcast.Broadcast import org.apache.spark.sql.{Column, DataFrame, Dataset, SparkSession} -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig +import org.apache.spark.sql.execution.datasources.mergetree.ClickHouseConfig import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.execution.metric.SQLMetrics.createMetric import org.apache.spark.sql.functions._ diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/mergetree/ClickHouseDataSource.scala similarity index 98% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala rename to backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/mergetree/ClickHouseDataSource.scala index 8c1062f4c7b6..96a018579f4e 100644 --- a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala +++ b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/mergetree/ClickHouseDataSource.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.execution.datasources.v2.clickhouse +package org.apache.spark.sql.execution.datasources.mergetree import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap @@ -133,13 +133,13 @@ class ClickHouseDataSource extends DeltaDataSource { } else { Map.empty } - (new ClickHouseTableV2( + new ClickHouseTableV2( sqlContext.sparkSession, new Path(maybePath), timeTravelOpt = timeTravelByParams, options = dfOptions, cdcOptions = new CaseInsensitiveStringMap(cdcOptions.asJava) - )).toBaseRelation + ).toBaseRelation } } } diff --git a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/mergetree/DeltaMergeTreeFileFormat.scala similarity index 93% rename from backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala rename to backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/mergetree/DeltaMergeTreeFileFormat.scala index 24dbc6e03bef..24757763b7e7 100644 --- a/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala +++ b/backends-clickhouse/src/main/delta-23/org/apache/spark/sql/execution/datasources/mergetree/DeltaMergeTreeFileFormat.scala @@ -14,14 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.execution.datasources.v2.clickhouse.source +package org.apache.spark.sql.execution.datasources.mergetree import org.apache.spark.sql.SparkSession import org.apache.spark.sql.delta.DeltaParquetFileFormat import org.apache.spark.sql.delta.actions.Metadata import org.apache.spark.sql.execution.datasources.{OutputWriter, OutputWriterFactory} -import org.apache.spark.sql.execution.datasources.mergetree.DeltaMetaReader -import org.apache.spark.sql.execution.datasources.v1.GlutenMergeTreeWriterInjects +import org.apache.spark.sql.execution.datasources.ch.GlutenMergeTreeWriterInjects import org.apache.spark.sql.types.StructType import org.apache.hadoop.mapreduce.{Job, TaskAttemptContext} diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala index 00940a4851e0..196b1e812f5b 100644 --- a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala +++ b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala @@ -33,8 +33,7 @@ import org.apache.spark.sql.delta.stats.DeltaJobStatisticsTracker import org.apache.spark.sql.execution.{CHDelayedCommitProtocol, QueryExecution, SparkPlan, SQLExecution} import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec import org.apache.spark.sql.execution.datasources.{BasicWriteJobStatsTracker, FakeRowAdaptor, FileFormatWriter, WriteFiles, WriteJobStatsTracker} -import org.apache.spark.sql.execution.datasources.v1.clickhouse.MergeTreeFileFormatWriter -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig +import org.apache.spark.sql.execution.datasources.mergetree.{ClickHouseConfig, MergeTreeFileFormatWriter} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.util.SerializableConfiguration diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/DeltaLog.scala b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/DeltaLog.scala index bac5231309b8..ce4694fa5b66 100644 --- a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/DeltaLog.scala +++ b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/DeltaLog.scala @@ -50,7 +50,7 @@ import org.apache.spark.sql.catalyst.expressions.{And, Attribute, Cast, Expressi import org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper import org.apache.spark.sql.catalyst.util.FailFastMode import org.apache.spark.sql.execution.datasources._ -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig +import org.apache.spark.sql.execution.datasources.mergetree.ClickHouseConfig import org.apache.spark.sql.expressions.UserDefinedFunction import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.sources.{BaseRelation, InsertableRelation} diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/Snapshot.scala b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/Snapshot.scala index 8836f7c88d23..a90848219f64 100644 --- a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/Snapshot.scala +++ b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/Snapshot.scala @@ -37,7 +37,7 @@ import org.apache.hadoop.fs.{FileStatus, Path} import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.expressions.Expression -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig +import org.apache.spark.sql.execution.datasources.mergetree.ClickHouseConfig import org.apache.spark.sql.functions._ import org.apache.spark.sql.types.StructType import org.apache.spark.util.Utils diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/catalog/ClickHouseSparkCatalog.scala similarity index 98% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala rename to backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/catalog/ClickHouseSparkCatalog.scala index dde7013962d0..35eca2276fd1 100644 --- a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala +++ b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/catalog/ClickHouseSparkCatalog.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.execution.datasources.v2.clickhouse +package org.apache.spark.sql.delta.catalog import org.apache.gluten.sql.shims.SparkShimLoader @@ -30,7 +30,6 @@ import org.apache.spark.sql.connector.expressions.{FieldReference, IdentityTrans import org.apache.spark.sql.connector.write.{LogicalWriteInfo, V1Write, WriteBuilder} import org.apache.spark.sql.delta.{DeltaConfigs, DeltaErrors, DeltaLog, DeltaOptions, DeltaTableUtils} import org.apache.spark.sql.delta.DeltaTableIdentifier.gluePermissionError -import org.apache.spark.sql.delta.catalog.{BucketTransform, ClickHouseTableV2, DeltaTableV2, TempClickHouseTableV2} import org.apache.spark.sql.delta.commands.{CreateDeltaTableCommand, TableCreationModes, WriteIntoDelta} import org.apache.spark.sql.delta.metering.DeltaLogging import org.apache.spark.sql.delta.skipping.clustering.ClusteredTableUtils @@ -38,7 +37,8 @@ import org.apache.spark.sql.delta.skipping.clustering.temp.{ClusterBySpec, Clust import org.apache.spark.sql.delta.sources.{DeltaSourceUtils, DeltaSQLConf} import org.apache.spark.sql.delta.stats.StatisticsCollection import org.apache.spark.sql.execution.datasources.{DataSource, PartitioningUtils} -import org.apache.spark.sql.execution.datasources.v2.clickhouse.utils.CHDataSourceUtils +import org.apache.spark.sql.execution.datasources.mergetree.ClickHouseConfig +import org.apache.spark.sql.execution.datasources.mergetree.utils.CHDataSourceUtils import org.apache.spark.sql.sources.InsertableRelation import org.apache.spark.sql.types.StructType @@ -679,7 +679,7 @@ class ClickHouseSparkCatalog override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = table match { case supportsWrite: SupportsWrite => supportsWrite.newWriteBuilder(info) - case _ => throw DeltaErrors.unsupportedWriteStagedTable(name) + case _ => throw DeltaErrors.unsupportedWriteStagedTable(name()) } } } diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala index 5f6a2dc3d712..ba5f42bb9dd0 100644 --- a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala +++ b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala @@ -30,10 +30,8 @@ import org.apache.spark.sql.delta.{ClickhouseSnapshot, DeltaErrors, DeltaLog, De import org.apache.spark.sql.delta.actions.{Metadata, Protocol} import org.apache.spark.sql.delta.sources.DeltaDataSource import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, PartitionDirectory} -import org.apache.spark.sql.execution.datasources.clickhouse.utils.MergeTreePartsPartitionsUtil -import org.apache.spark.sql.execution.datasources.mergetree.StorageMeta -import org.apache.spark.sql.execution.datasources.v2.clickhouse.source.DeltaMergeTreeFileFormat -import org.apache.spark.sql.execution.datasources.v2.clickhouse.utils.CHDataSourceUtils +import org.apache.spark.sql.execution.datasources.mergetree.{DeltaMergeTreeFileFormat, StorageMeta} +import org.apache.spark.sql.execution.datasources.mergetree.utils.{CHDataSourceUtils, MergeTreePartsPartitionsUtil} import org.apache.spark.sql.util.CaseInsensitiveStringMap import org.apache.spark.util.collection.BitSet diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala index 5b2170220228..9090a0cbbf0d 100644 --- a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala +++ b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala @@ -39,8 +39,8 @@ import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Expression} import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, UnaryNode} import org.apache.spark.sql.execution.command.RunnableCommand -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig -import org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata.AddMergeTreeParts +import org.apache.spark.sql.execution.datasources.mergetree.ClickHouseConfig +import org.apache.spark.sql.execution.datasources.mergetree.metadata.AddMergeTreeParts import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.execution.metric.SQLMetrics.createMetric import org.apache.spark.sql.types._ diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala index 0a3b2ef5ef26..dc2b0022fcc1 100644 --- a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala +++ b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala @@ -33,16 +33,16 @@ import org.apache.spark.sql.delta.actions.{AddFile, FileAction} import org.apache.spark.sql.delta.catalog.{ClickHouseTableV2, DeltaTableV2} import org.apache.spark.sql.errors.QueryExecutionErrors import org.apache.spark.sql.execution.datasources.CHDatasourceJniWrapper -import org.apache.spark.sql.execution.datasources.v1.CHMergeTreeWriterInjects -import org.apache.spark.sql.execution.datasources.v1.clickhouse._ -import org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata.{AddFileTags, AddMergeTreeParts} -import org.apache.spark.sql.execution.datasources.v2.clickhouse.utils.CHDataSourceUtils import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.StructType import org.apache.spark.util.{SerializableConfiguration, SystemClock, Utils} import org.apache.hadoop.fs.{FileAlreadyExistsException, Path} import org.apache.hadoop.mapreduce.{TaskAttemptContext, TaskAttemptID, TaskID, TaskType} import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl +import org.apache.spark.sql.execution.datasources.ch.CHMergeTreeWriterInjects +import org.apache.spark.sql.execution.datasources.mergetree.MergeTreeWriteTaskResult +import org.apache.spark.sql.execution.datasources.mergetree.metadata.{AddFileTags, AddMergeTreeParts} +import org.apache.spark.sql.execution.datasources.mergetree.utils.CHDataSourceUtils import java.util.Date import scala.collection.mutable.ArrayBuffer diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/VacuumCommand.scala b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/VacuumCommand.scala index 9f455fb27bb1..3de073dd2089 100644 --- a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/VacuumCommand.scala +++ b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/delta/commands/VacuumCommand.scala @@ -34,8 +34,8 @@ import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.spark.broadcast.Broadcast import org.apache.spark.paths.SparkPath +import org.apache.spark.sql.execution.datasources.mergetree.ClickHouseConfig import org.apache.spark.sql.{Column, DataFrame, Dataset, Encoder, SparkSession} -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.execution.metric.SQLMetrics.createMetric import org.apache.spark.sql.functions._ diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/mergetree/ClickHouseDataSource.scala similarity index 98% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala rename to backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/mergetree/ClickHouseDataSource.scala index fcf1cee66671..aeedebe6d832 100644 --- a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala +++ b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/mergetree/ClickHouseDataSource.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.execution.datasources.v2.clickhouse +package org.apache.spark.sql.execution.datasources.mergetree import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap @@ -133,12 +133,12 @@ class ClickHouseDataSource extends DeltaDataSource { } else { cdcOptions.toMap } - (new ClickHouseTableV2( + new ClickHouseTableV2( sqlContext.sparkSession, new Path(maybePath), timeTravelOpt = timeTravelByParams, options = dfOptions - )).toBaseRelation + ).toBaseRelation } } } diff --git a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/mergetree/DeltaMergeTreeFileFormat.scala similarity index 93% rename from backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala rename to backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/mergetree/DeltaMergeTreeFileFormat.scala index 6cc431f4f99c..b1a06bb2623a 100644 --- a/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala +++ b/backends-clickhouse/src/main/delta-32/org/apache/spark/sql/execution/datasources/mergetree/DeltaMergeTreeFileFormat.scala @@ -14,14 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.execution.datasources.v2.clickhouse.source +package org.apache.spark.sql.execution.datasources.mergetree import org.apache.spark.sql.SparkSession import org.apache.spark.sql.delta.DeltaParquetFileFormat import org.apache.spark.sql.delta.actions.{Metadata, Protocol} import org.apache.spark.sql.execution.datasources.{OutputWriter, OutputWriterFactory} -import org.apache.spark.sql.execution.datasources.mergetree.DeltaMetaReader -import org.apache.spark.sql.execution.datasources.v1.GlutenMergeTreeWriterInjects +import org.apache.spark.sql.execution.datasources.ch.GlutenMergeTreeWriterInjects import org.apache.spark.sql.types.StructType import org.apache.hadoop.mapreduce.{Job, TaskAttemptContext} diff --git a/backends-clickhouse/src/main/java/org/apache/gluten/utils/SnowflakeIdWorker.java b/backends-clickhouse/src/main/java/org/apache/gluten/utils/SnowflakeIdWorker.java index 23acaeec9e6d..a5ab4ea21705 100644 --- a/backends-clickhouse/src/main/java/org/apache/gluten/utils/SnowflakeIdWorker.java +++ b/backends-clickhouse/src/main/java/org/apache/gluten/utils/SnowflakeIdWorker.java @@ -19,7 +19,7 @@ import org.apache.gluten.exception.GlutenException; import org.apache.spark.SparkEnv; -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig; +import org.apache.spark.sql.execution.datasources.mergetree.ClickHouseConfig; /** * An object that generates IDs. This is broken into a separate class in case we ever want to diff --git a/backends-clickhouse/src/main/java/org/apache/spark/sql/execution/datasources/clickhouse/ExtensionTableBuilder.java b/backends-clickhouse/src/main/java/org/apache/spark/sql/execution/datasources/mergetree/ExtensionTableBuilder.java similarity index 90% rename from backends-clickhouse/src/main/java/org/apache/spark/sql/execution/datasources/clickhouse/ExtensionTableBuilder.java rename to backends-clickhouse/src/main/java/org/apache/spark/sql/execution/datasources/mergetree/ExtensionTableBuilder.java index a305972045b9..9125614814eb 100644 --- a/backends-clickhouse/src/main/java/org/apache/spark/sql/execution/datasources/clickhouse/ExtensionTableBuilder.java +++ b/backends-clickhouse/src/main/java/org/apache/spark/sql/execution/datasources/mergetree/ExtensionTableBuilder.java @@ -14,10 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.execution.datasources.clickhouse; - -import org.apache.spark.sql.execution.datasources.mergetree.MetaSerializer; -import org.apache.spark.sql.execution.datasources.mergetree.PartSerializer; +package org.apache.spark.sql.execution.datasources.mergetree; import java.util.List; import java.util.Map; diff --git a/backends-clickhouse/src/main/java/org/apache/spark/sql/execution/datasources/clickhouse/ExtensionTableNode.java b/backends-clickhouse/src/main/java/org/apache/spark/sql/execution/datasources/mergetree/ExtensionTableNode.java similarity index 97% rename from backends-clickhouse/src/main/java/org/apache/spark/sql/execution/datasources/clickhouse/ExtensionTableNode.java rename to backends-clickhouse/src/main/java/org/apache/spark/sql/execution/datasources/mergetree/ExtensionTableNode.java index bb04652be440..0868d4fac39c 100644 --- a/backends-clickhouse/src/main/java/org/apache/spark/sql/execution/datasources/clickhouse/ExtensionTableNode.java +++ b/backends-clickhouse/src/main/java/org/apache/spark/sql/execution/datasources/mergetree/ExtensionTableNode.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.execution.datasources.clickhouse; +package org.apache.spark.sql.execution.datasources.mergetree; import org.apache.gluten.backendsapi.BackendsApiManager; import org.apache.gluten.substrait.rel.SplitInfo; diff --git a/backends-clickhouse/src/main/java/org/apache/spark/sql/execution/datasources/clickhouse/MergeTreePartFilterReturnedRange.java b/backends-clickhouse/src/main/java/org/apache/spark/sql/execution/datasources/mergetree/MergeTreePartFilterReturnedRange.java similarity index 95% rename from backends-clickhouse/src/main/java/org/apache/spark/sql/execution/datasources/clickhouse/MergeTreePartFilterReturnedRange.java rename to backends-clickhouse/src/main/java/org/apache/spark/sql/execution/datasources/mergetree/MergeTreePartFilterReturnedRange.java index 357e34e06126..0647a70a4d1f 100644 --- a/backends-clickhouse/src/main/java/org/apache/spark/sql/execution/datasources/clickhouse/MergeTreePartFilterReturnedRange.java +++ b/backends-clickhouse/src/main/java/org/apache/spark/sql/execution/datasources/mergetree/MergeTreePartFilterReturnedRange.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.execution.datasources.clickhouse; +package org.apache.spark.sql.execution.datasources.mergetree; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/backends-clickhouse/src/main/java/org/apache/spark/sql/execution/datasources/clickhouse/WriteReturnedMetric.java b/backends-clickhouse/src/main/java/org/apache/spark/sql/execution/datasources/mergetree/WriteReturnedMetric.java similarity index 97% rename from backends-clickhouse/src/main/java/org/apache/spark/sql/execution/datasources/clickhouse/WriteReturnedMetric.java rename to backends-clickhouse/src/main/java/org/apache/spark/sql/execution/datasources/mergetree/WriteReturnedMetric.java index 223e92c28d1d..4c0c26085b40 100644 --- a/backends-clickhouse/src/main/java/org/apache/spark/sql/execution/datasources/clickhouse/WriteReturnedMetric.java +++ b/backends-clickhouse/src/main/java/org/apache/spark/sql/execution/datasources/mergetree/WriteReturnedMetric.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.execution.datasources.clickhouse; +package org.apache.spark.sql.execution.datasources.mergetree; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/backends-clickhouse/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister b/backends-clickhouse/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister index 1fff03368186..403382eef5d8 100644 --- a/backends-clickhouse/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister +++ b/backends-clickhouse/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister @@ -1 +1 @@ -org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseDataSource +org.apache.spark.sql.execution.datasources.mergetree.ClickHouseDataSource diff --git a/backends-clickhouse/src/main/resources/org/apache/spark/sql/execution/datasources/v1/write_optimization.proto b/backends-clickhouse/src/main/resources/org/apache/spark/sql/execution/datasources/ch/write_optimization.proto similarity index 99% rename from backends-clickhouse/src/main/resources/org/apache/spark/sql/execution/datasources/v1/write_optimization.proto rename to backends-clickhouse/src/main/resources/org/apache/spark/sql/execution/datasources/ch/write_optimization.proto index a09f3ea0940c..8ebb027b3604 100644 --- a/backends-clickhouse/src/main/resources/org/apache/spark/sql/execution/datasources/v1/write_optimization.proto +++ b/backends-clickhouse/src/main/resources/org/apache/spark/sql/execution/datasources/ch/write_optimization.proto @@ -3,7 +3,7 @@ syntax = "proto3"; package local_engine; -option java_package = "org.apache.spark.sql.execution.datasources.v1"; +option java_package = "org.apache.spark.sql.execution.datasources.ch"; option java_multiple_files = true; message Write { diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala index 6760d29561a3..9864082ae010 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala @@ -38,8 +38,7 @@ import org.apache.spark.sql.catalyst.catalog.ExternalCatalogUtils import org.apache.spark.sql.catalyst.util.{DateFormatter, TimestampFormatter} import org.apache.spark.sql.connector.read.InputPartition import org.apache.spark.sql.execution.datasources.FilePartition -import org.apache.spark.sql.execution.datasources.clickhouse.{ExtensionTableBuilder, ExtensionTableNode} -import org.apache.spark.sql.execution.datasources.mergetree.PartSerializer +import org.apache.spark.sql.execution.datasources.mergetree.{ExtensionTableBuilder, ExtensionTableNode, PartSerializer} import org.apache.spark.sql.execution.metric.SQLMetric import org.apache.spark.sql.types._ import org.apache.spark.sql.utils.SparkInputMetricsUtil.InputMetricsWrapper diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala index 16f5fa064c09..0ee9d6f07895 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala @@ -32,7 +32,7 @@ import org.apache.spark.internal.Logging import org.apache.spark.listener.CHGlutenSQLAppStatusListener import org.apache.spark.network.util.JavaUtils import org.apache.spark.rpc.{GlutenDriverEndpoint, GlutenExecutorEndpoint} -import org.apache.spark.sql.execution.datasources.v1._ +import org.apache.spark.sql.execution.datasources.ch._ import org.apache.spark.sql.utils.ExpressionUtil import org.apache.spark.util.SparkDirectoryUtil diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala index 903523791a1b..26520c37f56a 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala @@ -46,8 +46,8 @@ import org.apache.spark.sql.delta.files.TahoeFileIndex import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.adaptive.AQEShuffleReadExec import org.apache.spark.sql.execution.datasources.{FileFormat, HadoopFsRelation} +import org.apache.spark.sql.execution.datasources.mergetree.DeltaMergeTreeFileFormat import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat -import org.apache.spark.sql.execution.datasources.v2.clickhouse.source.DeltaMergeTreeFileFormat import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ShuffleExchangeExec} import org.apache.spark.sql.execution.joins.{BuildSideRelation, ClickHouseBuildSideRelation, HashedRelationBroadcastMode} import org.apache.spark.sql.execution.metric.SQLMetric diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHTransformerApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHTransformerApi.scala index e5b71825853b..dee5e2d434ad 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHTransformerApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHTransformerApi.scala @@ -31,7 +31,7 @@ import org.apache.spark.sql.delta.files.TahoeFileIndex import org.apache.spark.sql.execution.SparkPlan import org.apache.spark.sql.execution.aggregate.HashAggregateExec import org.apache.spark.sql.execution.datasources.{FileFormat, HadoopFsRelation, PartitionDirectory} -import org.apache.spark.sql.execution.datasources.v2.clickhouse.source.DeltaMergeTreeFileFormat +import org.apache.spark.sql.execution.datasources.mergetree.DeltaMergeTreeFileFormat import org.apache.spark.sql.sources.DataSourceRegister import org.apache.spark.sql.types._ import org.apache.spark.util.collection.BitSet diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/ClickhouseSnapshot.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/ClickhouseSnapshot.scala index e3f643046671..2a6a5ccc351e 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/ClickhouseSnapshot.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/ClickhouseSnapshot.scala @@ -22,7 +22,7 @@ import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.expressions.{AttributeReference, BindReferences, Expression, Predicate} import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.delta.stats.DeltaScan -import org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata.{AddFileTags, AddMergeTreeParts} +import org.apache.spark.sql.execution.datasources.mergetree.metadata.{AddFileTags, AddMergeTreeParts} import com.google.common.base.Objects import com.google.common.cache.{Cache, CacheBuilder, CacheLoader, LoadingCache} diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/commands/GlutenCHCacheDataCommand.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/commands/GlutenCHCacheDataCommand.scala index 1c7b4f232205..6419a0d4c4a4 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/commands/GlutenCHCacheDataCommand.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/commands/GlutenCHCacheDataCommand.scala @@ -27,9 +27,7 @@ import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, import org.apache.spark.sql.delta._ import org.apache.spark.sql.execution.command.LeafRunnableCommand import org.apache.spark.sql.execution.commands.GlutenCacheBase._ -import org.apache.spark.sql.execution.datasources.clickhouse.ExtensionTableBuilder -import org.apache.spark.sql.execution.datasources.mergetree.{PartSerializer, StorageMeta} -import org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata.AddMergeTreeParts +import org.apache.spark.sql.execution.datasources.mergetree._ import org.apache.spark.sql.types.{BooleanType, StringType} import org.apache.hadoop.fs.Path @@ -134,12 +132,12 @@ case class GlutenCHCacheDataCommand( } val executorIdsToAddFiles = - scala.collection.mutable.Map[String, ArrayBuffer[AddMergeTreeParts]]() + scala.collection.mutable.Map[String, ArrayBuffer[metadata.AddMergeTreeParts]]() val executorIdsToParts = scala.collection.mutable.Map[String, String]() - executorIdsToAddFiles.put(ALL_EXECUTORS, new ArrayBuffer[AddMergeTreeParts]()) + executorIdsToAddFiles.put(ALL_EXECUTORS, new ArrayBuffer[metadata.AddMergeTreeParts]()) selectedAddFiles.foreach( addFile => { - val mergeTreePart = addFile.asInstanceOf[AddMergeTreeParts] + val mergeTreePart = addFile.asInstanceOf[metadata.AddMergeTreeParts] val partName = mergeTreePart.name val tableUri = URI.create(mergeTreePart.tablePath) val relativeTablePath = if (tableUri.getPath.startsWith("/")) { @@ -156,7 +154,7 @@ case class GlutenCHCacheDataCommand( locations.foreach( executor => { if (!executorIdsToAddFiles.contains(executor)) { - executorIdsToAddFiles.put(executor, new ArrayBuffer[AddMergeTreeParts]()) + executorIdsToAddFiles.put(executor, new ArrayBuffer[metadata.AddMergeTreeParts]()) } executorIdsToAddFiles(executor).append(mergeTreePart) }) diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/CHBlockStripes.java b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/ch/CHBlockStripes.java similarity index 98% rename from backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/CHBlockStripes.java rename to backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/ch/CHBlockStripes.java index 560fbd503998..772fa4d3d3e1 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/CHBlockStripes.java +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/ch/CHBlockStripes.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.execution.datasources.v1; +package org.apache.spark.sql.execution.datasources.ch; import org.apache.gluten.row.SparkRowInfo; import org.apache.gluten.vectorized.CHBlockConverterJniWrapper; diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/CHFormatWriterInjects.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/ch/CHFormatWriterInjects.scala similarity index 98% rename from backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/CHFormatWriterInjects.scala rename to backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/ch/CHFormatWriterInjects.scala index 03ee31daba08..ad3d857271da 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/CHFormatWriterInjects.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/ch/CHFormatWriterInjects.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.execution.datasources.v1 +package org.apache.spark.sql.execution.datasources.ch import org.apache.gluten.execution.datasource.GlutenRowSplitter import org.apache.gluten.expression.ConverterUtils diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/CHMergeTreeWriterInjects.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/ch/CHMergeTreeWriterInjects.scala similarity index 95% rename from backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/CHMergeTreeWriterInjects.scala rename to backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/ch/CHMergeTreeWriterInjects.scala index d25758b0f703..10d67c5a5ce1 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/CHMergeTreeWriterInjects.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/ch/CHMergeTreeWriterInjects.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.execution.datasources.v1 +package org.apache.spark.sql.execution.datasources.ch import org.apache.gluten.expression.ConverterUtils import org.apache.gluten.substrait.`type`.ColumnTypeNode @@ -26,8 +26,7 @@ import org.apache.gluten.utils.ConfigUtil import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.execution.datasources.{CHDatasourceJniWrapper, OutputWriter} -import org.apache.spark.sql.execution.datasources.mergetree.{MetaSerializer, PartSerializer, StorageConfigProvider, StorageMeta} -import org.apache.spark.sql.execution.datasources.v1.clickhouse.MergeTreeOutputWriter +import org.apache.spark.sql.execution.datasources.mergetree._ import org.apache.spark.sql.types.StructType import com.google.common.collect.Lists diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/CHOrcWriterInjects.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/ch/CHOrcWriterInjects.scala similarity index 96% rename from backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/CHOrcWriterInjects.scala rename to backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/ch/CHOrcWriterInjects.scala index 7c791bcf87d2..ad9f08f03eba 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/CHOrcWriterInjects.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/ch/CHOrcWriterInjects.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.execution.datasources.v1 +package org.apache.spark.sql.execution.datasources.ch import org.apache.hadoop.mapreduce.TaskAttemptContext diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/CHParquetWriterInjects.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/ch/CHParquetWriterInjects.scala similarity index 97% rename from backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/CHParquetWriterInjects.scala rename to backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/ch/CHParquetWriterInjects.scala index 8c700c413346..555af3f4f26d 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/CHParquetWriterInjects.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/ch/CHParquetWriterInjects.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.execution.datasources.v1 +package org.apache.spark.sql.execution.datasources.ch import org.apache.gluten.GlutenConfig diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/GlutenMergeTreeWriterInjects.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/ch/GlutenMergeTreeWriterInjects.scala similarity index 95% rename from backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/GlutenMergeTreeWriterInjects.scala rename to backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/ch/GlutenMergeTreeWriterInjects.scala index 36d8481b1a8a..ceafe0e2763b 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/GlutenMergeTreeWriterInjects.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/ch/GlutenMergeTreeWriterInjects.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.execution.datasources.v1 +package org.apache.spark.sql.execution.datasources.ch import org.apache.gluten.execution.datasource.GlutenFormatWriterInjects diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseConfig.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/mergetree/ClickHouseConfig.scala similarity index 96% rename from backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseConfig.scala rename to backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/mergetree/ClickHouseConfig.scala index 38f4fe7e26ab..df3cb15dfc48 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseConfig.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/mergetree/ClickHouseConfig.scala @@ -14,12 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.execution.datasources.v2.clickhouse +package org.apache.spark.sql.execution.datasources.mergetree import org.apache.gluten.backendsapi.clickhouse.CHConf import org.apache.spark.sql.catalyst.catalog.BucketSpec -import org.apache.spark.sql.execution.datasources.mergetree.StorageMeta import java.util diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeFileFormatDataWriter.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/mergetree/MergeTreeFileFormatDataWriter.scala similarity index 99% rename from backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeFileFormatDataWriter.scala rename to backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/mergetree/MergeTreeFileFormatDataWriter.scala index 29f2b7e16ec8..177cd9491d5d 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeFileFormatDataWriter.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/mergetree/MergeTreeFileFormatDataWriter.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.execution.datasources.v1.clickhouse +package org.apache.spark.sql.execution.datasources.mergetree import org.apache.gluten.execution.datasource.GlutenRowSplitter diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeFileFormatWriter.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/mergetree/MergeTreeFileFormatWriter.scala similarity index 99% rename from backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeFileFormatWriter.scala rename to backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/mergetree/MergeTreeFileFormatWriter.scala index f1489b86b3a1..b4be68902ea5 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeFileFormatWriter.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/mergetree/MergeTreeFileFormatWriter.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.execution.datasources.v1.clickhouse +package org.apache.spark.sql.execution.datasources.mergetree import org.apache.gluten.memory.CHThreadGroup @@ -33,7 +33,7 @@ import org.apache.spark.sql.delta.constraints.Constraint import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.datasources._ import org.apache.spark.sql.execution.datasources.FileFormatWriter.{processStats, ConcurrentOutputWriterSpec, OutputSpec} -import org.apache.spark.sql.execution.datasources.v1.GlutenMergeTreeWriterInjects +import org.apache.spark.sql.execution.datasources.ch.GlutenMergeTreeWriterInjects import org.apache.spark.sql.internal.SQLConf import org.apache.spark.util.{SerializableConfiguration, Utils} diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeOutputWriter.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/mergetree/MergeTreeOutputWriter.scala similarity index 93% rename from backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeOutputWriter.scala rename to backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/mergetree/MergeTreeOutputWriter.scala index 14ac659deabd..3b9e02695c90 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v1/clickhouse/MergeTreeOutputWriter.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/mergetree/MergeTreeOutputWriter.scala @@ -14,14 +14,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.execution.datasources.v1.clickhouse +package org.apache.spark.sql.execution.datasources.mergetree import org.apache.gluten.vectorized.CHColumnVector import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.delta.actions.AddFile import org.apache.spark.sql.execution.datasources.{CHDatasourceJniWrapper, FakeRow, OutputWriter} -import org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata.AddFileTags +import org.apache.spark.sql.execution.datasources.mergetree.metadata.AddFileTags import org.apache.spark.util.Utils import scala.collection.mutable.ArrayBuffer diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/mergetree/MetaSerializer.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/mergetree/MetaSerializer.scala index 94a74bd2fa15..dcf86e5d2528 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/mergetree/MetaSerializer.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/mergetree/MetaSerializer.scala @@ -19,9 +19,6 @@ package org.apache.spark.sql.execution.datasources.mergetree import org.apache.gluten.execution.MergeTreePartSplit import org.apache.gluten.expression.ConverterUtils -import org.apache.spark.sql.execution.datasources.clickhouse.ExtensionTableNode -import org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata.AddMergeTreeParts - import com.fasterxml.jackson.databind.ObjectMapper import io.substrait.proto.ReadRel @@ -61,7 +58,7 @@ object PartSerializer { PartSerializer(partList, starts, lengths) } - def fromAddMergeTreeParts(parts: Seq[AddMergeTreeParts]): PartSerializer = { + def fromAddMergeTreeParts(parts: Seq[metadata.AddMergeTreeParts]): PartSerializer = { val partList = parts.map(_.name) val starts = parts.map(_ => 0L) val lengths = parts.map(_.marks) diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/metadata/AddFileTags.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/mergetree/metadata/AddFileTags.scala similarity index 98% rename from backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/metadata/AddFileTags.scala rename to backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/mergetree/metadata/AddFileTags.scala index 8c610929a3d8..c1928bade78d 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/metadata/AddFileTags.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/mergetree/metadata/AddFileTags.scala @@ -14,10 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata +package org.apache.spark.sql.execution.datasources.mergetree.metadata import org.apache.spark.sql.delta.actions.AddFile -import org.apache.spark.sql.execution.datasources.clickhouse.WriteReturnedMetric +import org.apache.spark.sql.execution.datasources.mergetree.WriteReturnedMetric import com.fasterxml.jackson.core.`type`.TypeReference import com.fasterxml.jackson.databind.ObjectMapper diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/utils/CHDataSourceUtils.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/mergetree/utils/CHDataSourceUtils.scala similarity index 97% rename from backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/utils/CHDataSourceUtils.scala rename to backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/mergetree/utils/CHDataSourceUtils.scala index e2e7ac491ef2..0dbbf9ee31cf 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/utils/CHDataSourceUtils.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/mergetree/utils/CHDataSourceUtils.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.execution.datasources.v2.clickhouse.utils +package org.apache.spark.sql.execution.datasources.mergetree.utils import org.apache.spark.internal.Logging import org.apache.spark.sql.{AnalysisException, SparkSession} @@ -22,7 +22,7 @@ import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.connector.catalog.Identifier import org.apache.spark.sql.delta.DeltaTableIdentifier.gluePermissionError -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig +import org.apache.spark.sql.execution.datasources.mergetree.ClickHouseConfig import org.apache.hadoop.fs.Path diff --git a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/clickhouse/utils/MergeTreePartsPartitionsUtil.scala b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/mergetree/utils/MergeTreePartsPartitionsUtil.scala similarity index 97% rename from backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/clickhouse/utils/MergeTreePartsPartitionsUtil.scala rename to backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/mergetree/utils/MergeTreePartsPartitionsUtil.scala index 9347d8679b98..28e9e48ac45f 100644 --- a/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/clickhouse/utils/MergeTreePartsPartitionsUtil.scala +++ b/backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/datasources/mergetree/utils/MergeTreePartsPartitionsUtil.scala @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.spark.sql.execution.datasources.clickhouse.utils +package org.apache.spark.sql.execution.datasources.mergetree.utils import org.apache.gluten.backendsapi.clickhouse.{CHBackendSettings, CHConf} import org.apache.gluten.execution.{GlutenMergeTreePartition, MergeTreePartRange, MergeTreePartSplit} @@ -35,10 +35,7 @@ import org.apache.spark.sql.delta.ClickhouseSnapshot import org.apache.spark.sql.delta.catalog.ClickHouseTableV2 import org.apache.spark.sql.delta.files.TahoeFileIndex import org.apache.spark.sql.execution.datasources.{CHDatasourceJniWrapper, HadoopFsRelation, PartitionDirectory} -import org.apache.spark.sql.execution.datasources.clickhouse.{ExtensionTableBuilder, MergeTreePartFilterReturnedRange} -import org.apache.spark.sql.execution.datasources.mergetree.{PartSerializer, StorageMeta} -import org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata.AddMergeTreeParts -import org.apache.spark.sql.execution.datasources.v2.clickhouse.source.DeltaMergeTreeFileFormat +import org.apache.spark.sql.execution.datasources.mergetree._ import org.apache.spark.sql.types.BooleanType import org.apache.spark.util.SparkResourceUtil import org.apache.spark.util.collection.BitSet @@ -558,7 +555,7 @@ object MergeTreePartsPartitionsUtil extends Logging { } private def getMergeTreePartRange( - selectPartsFiles: Seq[AddMergeTreeParts], + selectPartsFiles: Seq[metadata.AddMergeTreeParts], snapshotId: String, database: String, tableName: String, diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/RunTPCHTest.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/RunTPCHTest.scala index 4c774c9e037b..6daae31bfc53 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/RunTPCHTest.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/RunTPCHTest.scala @@ -20,7 +20,7 @@ import org.apache.gluten.backendsapi.clickhouse.CHConf import org.apache.gluten.benchmarks.GenTPCHTableScripts import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig +import org.apache.spark.sql.execution.datasources.mergetree.ClickHouseConfig import org.apache.commons.io.FileUtils diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseJoinSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseJoinSuite.scala index 97ba1fe4214c..0eeec15544ae 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseJoinSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseJoinSuite.scala @@ -21,7 +21,7 @@ import org.apache.gluten.backendsapi.clickhouse.CHConf import org.apache.gluten.utils.UTSystemParameters import org.apache.spark.SparkConf -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig +import org.apache.spark.sql.execution.datasources.mergetree.ClickHouseConfig class GlutenClickHouseJoinSuite extends GlutenClickHouseWholeStageTransformerSuite { diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseSyntheticDataSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseSyntheticDataSuite.scala index bd739a291e51..bc687ba73de5 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseSyntheticDataSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseSyntheticDataSuite.scala @@ -23,7 +23,7 @@ import org.apache.spark.SparkConf import org.apache.spark.internal.Logging import org.apache.spark.sql.Row import org.apache.spark.sql.delta.DeltaLog -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig +import org.apache.spark.sql.execution.datasources.mergetree.ClickHouseConfig import java.time.LocalDate diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSAbstractSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSAbstractSuite.scala index f1d790322fe6..a3361d8565de 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSAbstractSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCDSAbstractSuite.scala @@ -24,7 +24,7 @@ import org.apache.spark.SparkConf import org.apache.spark.internal.Logging import org.apache.spark.sql.{DataFrame, Row, SparkSession} import org.apache.spark.sql.delta.{ClickhouseSnapshot, DeltaLog} -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig +import org.apache.spark.sql.execution.datasources.mergetree.ClickHouseConfig import org.apache.spark.sql.types.{StructField, StructType} import org.apache.commons.io.FileUtils diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHAbstractSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHAbstractSuite.scala index 7f15edf73b85..41593d9773ed 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHAbstractSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseTPCHAbstractSuite.scala @@ -23,7 +23,7 @@ import org.apache.spark.SparkConf import org.apache.spark.internal.Logging import org.apache.spark.sql.DataFrame import org.apache.spark.sql.delta.{ClickhouseSnapshot, DeltaLog} -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig +import org.apache.spark.sql.execution.datasources.mergetree.ClickHouseConfig import org.apache.commons.io.FileUtils import org.scalatest.time.SpanSugar.convertIntToGrainOfTime diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseWholeStageTransformerSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseWholeStageTransformerSuite.scala index e5c4d14a3442..69e6793668e5 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseWholeStageTransformerSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseWholeStageTransformerSuite.scala @@ -20,7 +20,7 @@ import org.apache.gluten.GlutenConfig import org.apache.gluten.utils.UTSystemParameters import org.apache.spark.{SPARK_VERSION_SHORT, SparkConf} -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig +import org.apache.spark.sql.execution.datasources.mergetree.ClickHouseConfig import org.apache.commons.io.FileUtils import org.scalatest.Tag diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala index b268eb7192d3..55204a242dd4 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenFunctionValidateSuite.scala @@ -22,7 +22,7 @@ import org.apache.gluten.utils.UTSystemParameters import org.apache.spark.SparkConf import org.apache.spark.sql.{DataFrame, Row, TestUtils} import org.apache.spark.sql.catalyst.optimizer.{ConstantFolding, NullPropagation} -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig +import org.apache.spark.sql.execution.datasources.mergetree.ClickHouseConfig import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/compatibility/GlutenClickhouseFunctionSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/compatibility/GlutenClickhouseFunctionSuite.scala index f16c897671b1..2a4ef6e82a9a 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/compatibility/GlutenClickhouseFunctionSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/compatibility/GlutenClickhouseFunctionSuite.scala @@ -21,7 +21,7 @@ import org.apache.gluten.execution.GlutenClickHouseTPCHAbstractSuite import org.apache.gluten.utils.UTSystemParameters import org.apache.spark.SparkConf -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig +import org.apache.spark.sql.execution.datasources.mergetree.ClickHouseConfig class GlutenClickhouseFunctionSuite extends GlutenClickHouseTPCHAbstractSuite { override protected val needCopyParquetToTablePath = true diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseHiveTableSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseHiveTableSuite.scala index 4e2b5ad63e0a..4e28dbbd1093 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseHiveTableSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseHiveTableSuite.scala @@ -25,7 +25,7 @@ import org.apache.spark.SparkConf import org.apache.spark.sql.{DataFrame, SaveMode} import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig +import org.apache.spark.sql.execution.datasources.mergetree.ClickHouseConfig import org.apache.spark.sql.hive.HiveTableScanExecTransformer import org.apache.spark.sql.internal.SQLConf diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseNativeWriteTableSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseNativeWriteTableSuite.scala index 53aef16d143e..494564b556b3 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseNativeWriteTableSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseNativeWriteTableSuite.scala @@ -25,7 +25,7 @@ import org.apache.spark.SparkConf import org.apache.spark.gluten.NativeWriteChecker import org.apache.spark.sql.delta.DeltaLog import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig +import org.apache.spark.sql.execution.datasources.mergetree.ClickHouseConfig import org.apache.spark.sql.types._ import scala.reflect.runtime.universe.TypeTag diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeCacheDataSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeCacheDataSuite.scala index bf3be1e52907..6f5ad90f6683 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeCacheDataSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeCacheDataSuite.scala @@ -22,7 +22,7 @@ import org.apache.gluten.execution.{FileSourceScanExecTransformer, GlutenClickHo import org.apache.spark.SparkConf import org.apache.spark.sql.delta.files.TahoeFileIndex import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper -import org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata.AddMergeTreeParts +import org.apache.spark.sql.execution.datasources.mergetree.metadata.AddMergeTreeParts import org.apache.commons.io.FileUtils import org.apache.hadoop.conf.Configuration diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreePathBasedWriteSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreePathBasedWriteSuite.scala index 62b9ee3bcb31..966a9068f5a3 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreePathBasedWriteSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreePathBasedWriteSuite.scala @@ -26,7 +26,7 @@ import org.apache.spark.sql.delta.files.TahoeFileIndex import org.apache.spark.sql.execution.LocalTableScanExec import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.execution.datasources.mergetree.StorageMeta -import org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata.AddMergeTreeParts +import org.apache.spark.sql.execution.datasources.mergetree.metadata.AddMergeTreeParts import org.apache.spark.sql.functions._ import io.delta.tables.ClickhouseTable diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteOnHDFSSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteOnHDFSSuite.scala index ca1c9eef2a77..26fb46ca7f0f 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteOnHDFSSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteOnHDFSSuite.scala @@ -24,7 +24,7 @@ import org.apache.spark.sql.SaveMode import org.apache.spark.sql.delta.catalog.ClickHouseTableV2 import org.apache.spark.sql.delta.files.TahoeFileIndex import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper -import org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata.AddMergeTreeParts +import org.apache.spark.sql.execution.datasources.mergetree.metadata.AddMergeTreeParts import org.apache.commons.io.FileUtils import org.apache.hadoop.conf.Configuration diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteOnHDFSWithRocksDBMetaSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteOnHDFSWithRocksDBMetaSuite.scala index 26c6dd852b89..ae61808efc9c 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteOnHDFSWithRocksDBMetaSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteOnHDFSWithRocksDBMetaSuite.scala @@ -24,7 +24,7 @@ import org.apache.spark.sql.SaveMode import org.apache.spark.sql.delta.catalog.ClickHouseTableV2 import org.apache.spark.sql.delta.files.TahoeFileIndex import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper -import org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata.AddMergeTreeParts +import org.apache.spark.sql.execution.datasources.mergetree.metadata.AddMergeTreeParts import org.apache.commons.io.FileUtils import org.apache.hadoop.conf.Configuration diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteOnS3Suite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteOnS3Suite.scala index dfdf02d45564..18e5c4530880 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteOnS3Suite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteOnS3Suite.scala @@ -24,7 +24,7 @@ import org.apache.spark.sql.SaveMode import org.apache.spark.sql.delta.catalog.ClickHouseTableV2 import org.apache.spark.sql.delta.files.TahoeFileIndex import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper -import org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata.AddMergeTreeParts +import org.apache.spark.sql.execution.datasources.mergetree.metadata.AddMergeTreeParts import _root_.org.apache.commons.io.FileUtils import io.minio._ diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteSuite.scala index 186078c18dd1..8d3c30971941 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteSuite.scala @@ -26,7 +26,7 @@ import org.apache.spark.sql.delta.catalog.ClickHouseTableV2 import org.apache.spark.sql.delta.files.TahoeFileIndex import org.apache.spark.sql.execution.LocalTableScanExec import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper -import org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata.AddMergeTreeParts +import org.apache.spark.sql.execution.datasources.mergetree.metadata.AddMergeTreeParts import org.apache.commons.io.filefilter.WildcardFileFilter diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickhouseMergetreeSoftAffinitySuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickhouseMergetreeSoftAffinitySuite.scala index aead0bf47fa3..45f2e2395f48 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickhouseMergetreeSoftAffinitySuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickhouseMergetreeSoftAffinitySuite.scala @@ -22,7 +22,7 @@ import org.apache.gluten.execution.{GlutenClickHouseTPCHAbstractSuite, GlutenMer import org.apache.spark.sql.connector.read.InputPartition import org.apache.spark.sql.delta.catalog.ClickHouseTableV2 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper -import org.apache.spark.sql.execution.datasources.clickhouse.utils.MergeTreePartsPartitionsUtil +import org.apache.spark.sql.execution.datasources.mergetree.utils.MergeTreePartsPartitionsUtil import org.apache.hadoop.fs.Path diff --git a/cpp-ch/local-engine/proto/write_optimization.proto b/cpp-ch/local-engine/proto/write_optimization.proto index d1338a75fedb..f2a44cb91c26 120000 --- a/cpp-ch/local-engine/proto/write_optimization.proto +++ b/cpp-ch/local-engine/proto/write_optimization.proto @@ -1 +1 @@ -../../../backends-clickhouse/src/main/resources/org/apache/spark/sql/execution/datasources/v1/write_optimization.proto \ No newline at end of file +../../../backends-clickhouse/src/main/resources/org/apache/spark/sql/execution/datasources/ch/write_optimization.proto \ No newline at end of file