moving stuff

ericm-db · Oct 31, 2024 · b45be20 · b45be20
1 parent d39092a
commit b45be20
Show file tree

Hide file tree

Showing 9 changed files with 28 additions and 16 deletions.
diff --git a/...che/spark/sql/avro/AvroDeserializer.scala → ...che/spark/sql/avro/AvroDeserializer.scala b/...che/spark/sql/avro/AvroDeserializer.scala → ...che/spark/sql/avro/AvroDeserializer.scala
@@ -19,20 +19,22 @@ package org.apache.spark.sql.avro
 
 import java.math.BigDecimal
 import java.nio.ByteBuffer
+
 import scala.jdk.CollectionConverters._
+
+import org.apache.avro.{LogicalTypes, Schema, SchemaBuilder}
 import org.apache.avro.Conversions.DecimalConversion
 import org.apache.avro.LogicalTypes.{LocalTimestampMicros, LocalTimestampMillis, TimestampMicros, TimestampMillis}
 import org.apache.avro.Schema.Type._
 import org.apache.avro.generic._
 import org.apache.avro.util.Utf8
-import org.apache.avro.{LogicalTypes, Schema, SchemaBuilder}
 
-import org.apache.spark.sql.avro.AvroUtils.{AvroMatchedField, nonNullUnionBranches, toFieldStr}
+import org.apache.spark.sql.avro.AvroUtils.{nonNullUnionBranches, toFieldStr, AvroMatchedField}
+import org.apache.spark.sql.catalyst.{InternalRow, NoopFilters, StructFilters}
 import org.apache.spark.sql.catalyst.expressions.{SpecificInternalRow, UnsafeArrayData}
+import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, ArrayData, DateTimeUtils, GenericArrayData}
 import org.apache.spark.sql.catalyst.util.DateTimeConstants.MILLIS_PER_DAY
 import org.apache.spark.sql.catalyst.util.RebaseDateTime.RebaseSpec
-import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, ArrayData, DateTimeUtils, GenericArrayData}
-import org.apache.spark.sql.catalyst.{InternalRow, NoopFilters, StructFilters}
 import org.apache.spark.sql.errors.QueryCompilationErrors
 import org.apache.spark.sql.execution.datasources.DataSourceUtils
 import org.apache.spark.sql.internal.{LegacyBehaviorPolicy, SQLConf}

diff --git a/...pache/spark/sql/avro/AvroFileFormat.scala → ...pache/spark/sql/avro/AvroFileFormat.scala b/...pache/spark/sql/avro/AvroFileFormat.scala → ...pache/spark/sql/avro/AvroFileFormat.scala
@@ -18,11 +18,13 @@
 package org.apache.spark.sql.avro
 
 import java.io._
+
 import scala.util.control.NonFatal
+
+import org.apache.avro.{LogicalType, LogicalTypes, Schema}
 import org.apache.avro.file.DataFileReader
 import org.apache.avro.generic.{GenericDatumReader, GenericRecord}
 import org.apache.avro.mapred.FsInput
-import org.apache.avro.{LogicalType, LogicalTypes, Schema}
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.{FileStatus, Path}
 import org.apache.hadoop.mapreduce.Job

diff --git a/...g/apache/spark/sql/avro/AvroOptions.scala → ...g/apache/spark/sql/avro/AvroOptions.scala b/...g/apache/spark/sql/avro/AvroOptions.scala → ...g/apache/spark/sql/avro/AvroOptions.scala
@@ -18,14 +18,15 @@
 package org.apache.spark.sql.avro
 
 import java.net.URI
+
 import org.apache.avro.Schema
 import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.{FileSystem, Path}
 
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, FailFastMode, ParseMode}
 import org.apache.spark.sql.catalyst.{DataSourceOptions, FileSourceOptions}
+import org.apache.spark.sql.catalyst.util.{CaseInsensitiveMap, FailFastMode, ParseMode}
 import org.apache.spark.sql.errors.QueryCompilationErrors
 import org.apache.spark.sql.internal.SQLConf
 

diff --git a/...che/spark/sql/avro/AvroOutputWriter.scala → ...che/spark/sql/avro/AvroOutputWriter.scala b/...che/spark/sql/avro/AvroOutputWriter.scala → ...che/spark/sql/avro/AvroOutputWriter.scala
@@ -18,7 +18,9 @@
 package org.apache.spark.sql.avro
 
 import java.io.{IOException, OutputStream}
+
 import scala.jdk.CollectionConverters._
+
 import org.apache.avro.Schema
 import org.apache.avro.generic.GenericRecord
 import org.apache.avro.mapred.AvroKey
@@ -27,12 +29,11 @@ import org.apache.hadoop.io.NullWritable
 import org.apache.hadoop.mapreduce.{RecordWriter, TaskAttemptContext}
 
 import org.apache.spark.SPARK_VERSION_SHORT
+import org.apache.spark.sql.{SPARK_LEGACY_DATETIME_METADATA_KEY, SPARK_TIMEZONE_METADATA_KEY, SPARK_VERSION_METADATA_KEY}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.execution.datasources.OutputWriter
 import org.apache.spark.sql.internal.{LegacyBehaviorPolicy, SQLConf}
 import org.apache.spark.sql.types._
-import org.apache.spark.sql.{SPARK_LEGACY_DATETIME_METADATA_KEY, SPARK_TIMEZONE_METADATA_KEY, SPARK_VERSION_METADATA_KEY}
-
 // NOTE: This class is instantiated and used on executor side only, no need to be serializable.
 private[avro] class AvroOutputWriter(
     val path: String,

diff --git a/...rk/sql/avro/AvroOutputWriterFactory.scala → ...rk/sql/avro/AvroOutputWriterFactory.scala b/...rk/sql/avro/AvroOutputWriterFactory.scala → ...rk/sql/avro/AvroOutputWriterFactory.scala
diff --git a/...pache/spark/sql/avro/AvroSerializer.scala → ...pache/spark/sql/avro/AvroSerializer.scala b/...pache/spark/sql/avro/AvroSerializer.scala → ...pache/spark/sql/avro/AvroSerializer.scala
@@ -18,17 +18,19 @@
 package org.apache.spark.sql.avro
 
 import java.nio.ByteBuffer
+
 import scala.jdk.CollectionConverters._
+
+import org.apache.avro.{LogicalTypes, Schema}
 import org.apache.avro.Conversions.DecimalConversion
 import org.apache.avro.LogicalTypes.{LocalTimestampMicros, LocalTimestampMillis, TimestampMicros, TimestampMillis}
-import org.apache.avro.{LogicalTypes, Schema}
 import org.apache.avro.Schema.Type
 import org.apache.avro.Schema.Type._
 import org.apache.avro.generic.GenericData.{EnumSymbol, Fixed, Record}
 import org.apache.avro.util.Utf8
 
 import org.apache.spark.internal.Logging
-import org.apache.spark.sql.avro.AvroUtils.{AvroMatchedField, nonNullUnionBranches, toFieldStr}
+import org.apache.spark.sql.avro.AvroUtils.{nonNullUnionBranches, toFieldStr, AvroMatchedField}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{SpecializedGetters, SpecificInternalRow}
 import org.apache.spark.sql.catalyst.util.DateTimeUtils

diff --git a/...org/apache/spark/sql/avro/AvroUtils.scala → ...org/apache/spark/sql/avro/AvroUtils.scala b/...org/apache/spark/sql/avro/AvroUtils.scala → ...org/apache/spark/sql/avro/AvroUtils.scala
@@ -18,7 +18,9 @@ package org.apache.spark.sql.avro
 
 import java.io.{FileNotFoundException, IOException}
 import java.util.Locale
+
 import scala.jdk.CollectionConverters._
+
 import org.apache.avro.Schema
 import org.apache.avro.file.{DataFileReader, FileReader}
 import org.apache.avro.generic.{GenericDatumReader, GenericRecord}
@@ -28,18 +30,18 @@ import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.FileStatus
 import org.apache.hadoop.mapreduce.Job
 
-import org.apache.spark.internal.LogKeys.{CODEC_LEVEL, CODEC_NAME, CONFIG, PATH}
+import org.apache.spark.{SparkException, SparkIllegalArgumentException}
 import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.LogKeys.{CODEC_LEVEL, CODEC_NAME, CONFIG, PATH}
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.avro.AvroCompressionCodec._
 import org.apache.spark.sql.avro.AvroOptions.IGNORE_EXTENSION
-import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
 import org.apache.spark.sql.catalyst.{FileSourceOptions, InternalRow}
+import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap
 import org.apache.spark.sql.execution.datasources.OutputWriterFactory
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
 import org.apache.spark.util.Utils
-import org.apache.spark.{SparkException, SparkIllegalArgumentException}
 
 private[sql] object AvroUtils extends Logging {
   def inferSchema(

diff --git a/...apache/spark/sql/avro/CustomDecimal.scala → ...apache/spark/sql/avro/CustomDecimal.scala b/...apache/spark/sql/avro/CustomDecimal.scala → ...apache/spark/sql/avro/CustomDecimal.scala
diff --git a/...che/spark/sql/avro/SchemaConverters.scala → ...che/spark/sql/avro/SchemaConverters.scala b/...che/spark/sql/avro/SchemaConverters.scala → ...che/spark/sql/avro/SchemaConverters.scala
@@ -18,19 +18,21 @@
 package org.apache.spark.sql.avro
 
 import java.util.Locale
+
 import scala.collection.mutable
 import scala.jdk.CollectionConverters._
+
+import org.apache.avro.{LogicalTypes, Schema, SchemaBuilder}
 import org.apache.avro.LogicalTypes.{Decimal, _}
 import org.apache.avro.Schema.Type._
-import org.apache.avro.{LogicalTypes, Schema, SchemaBuilder}
 
 import org.apache.spark.annotation.DeveloperApi
-import org.apache.spark.internal.LogKeys.{FIELD_NAME, FIELD_TYPE, RECURSIVE_DEPTH}
 import org.apache.spark.internal.{Logging, MDC}
+import org.apache.spark.internal.LogKeys.{FIELD_NAME, FIELD_TYPE, RECURSIVE_DEPTH}
 import org.apache.spark.sql.avro.AvroOptions.RECURSIVE_FIELD_MAX_DEPTH_LIMIT
 import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
-import org.apache.spark.sql.types.Decimal.minBytesForPrecision
 import org.apache.spark.sql.types._
+import org.apache.spark.sql.types.Decimal.minBytesForPrecision
 
 /**
  * This object contains method that are used to convert sparkSQL schemas to avro schemas and vice