From bfdbfa856ed0c79d53a16799c1c596f0cb8c83d7 Mon Sep 17 00:00:00 2001 From: Vimal-Das Date: Thu, 19 May 2016 23:54:48 -0500 Subject: [PATCH] Cleaned Thrift Files --- .../ThriftWrapperSchemaConverterImpl.java | 123 ---------------- .../schema/table/column/CarbonColumn.java | 8 - .../schema/table/column/ColumnSchema.java | 21 --- .../query/util/DataFileFooterConverter.java | 63 -------- format/src/main/thrift/schema.thrift | 139 +----------------- 5 files changed, 6 insertions(+), 348 deletions(-) diff --git a/core/src/main/java/org/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImpl.java b/core/src/main/java/org/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImpl.java index 8b461d103b8..cb0afa5ff4f 100644 --- a/core/src/main/java/org/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImpl.java +++ b/core/src/main/java/org/carbondata/core/carbon/metadata/converter/ThriftWrapperSchemaConverterImpl.java @@ -21,7 +21,6 @@ import java.util.ArrayList; import java.util.List; -import org.carbondata.core.carbon.metadata.datatype.ConvertedType; import org.carbondata.core.carbon.metadata.datatype.DataType; import org.carbondata.core.carbon.metadata.encoder.Encoding; import org.carbondata.core.carbon.metadata.schema.SchemaEvolution; @@ -77,65 +76,6 @@ public org.carbondata.format.SchemaEvolutionEntry fromWrapperToExternalSchemaEvo return new org.carbondata.format.SchemaEvolution(thriftSchemaEvolEntryList); } - /** - * converts wrapper to external converter type - * - * @param convertedType - * @return - */ - private org.carbondata.format.ConvertedType fromWrapperToExternalConvertedType( - ConvertedType convertedType) { - - if (null == convertedType) { - return null; - } - switch (convertedType) { - case UTF8: - return org.carbondata.format.ConvertedType.UTF8; - case MAP: - return org.carbondata.format.ConvertedType.MAP; - case MAP_KEY_VALUE: - return org.carbondata.format.ConvertedType.MAP_KEY_VALUE; - case LIST: - return org.carbondata.format.ConvertedType.LIST; - case ENUM: - return org.carbondata.format.ConvertedType.ENUM; - case DECIMAL: - return org.carbondata.format.ConvertedType.DECIMAL; - case DATE: - return org.carbondata.format.ConvertedType.DATE; - case TIME_MILLIS: - return org.carbondata.format.ConvertedType.TIME_MILLIS; - case TIMESTAMP_MILLIS: - return org.carbondata.format.ConvertedType.TIMESTAMP_MILLIS; - case RESERVED: - return org.carbondata.format.ConvertedType.RESERVED; - case UINT_8: - return org.carbondata.format.ConvertedType.UINT_8; - case UINT_16: - return org.carbondata.format.ConvertedType.UINT_16; - case UINT_32: - return org.carbondata.format.ConvertedType.UINT_32; - case UINT_64: - return org.carbondata.format.ConvertedType.UINT_64; - case INT_8: - return org.carbondata.format.ConvertedType.INT_8; - case INT_16: - return org.carbondata.format.ConvertedType.INT_16; - case INT_32: - return org.carbondata.format.ConvertedType.INT_32; - case INT_64: - return org.carbondata.format.ConvertedType.INT_64; - case JSON: - return org.carbondata.format.ConvertedType.JSON; - case BSON: - return org.carbondata.format.ConvertedType.BSON; - case INTERVAL: - return org.carbondata.format.ConvertedType.INTERVAL; - default: - return org.carbondata.format.ConvertedType.UTF8; - } - } /** * converts from wrapper to external encoding @@ -212,8 +152,6 @@ private org.carbondata.format.DataType fromWrapperToExternalDataType(DataType da fromWrapperToExternalDataType(wrapperColumnSchema.getDataType()), wrapperColumnSchema.getColumnName(), wrapperColumnSchema.getColumnUniqueId(), wrapperColumnSchema.isColumnar(), encoders, wrapperColumnSchema.isDimensionColumn()); - thriftColumnSchema.setConverted_type( - fromWrapperToExternalConvertedType(wrapperColumnSchema.getConvertedType())); thriftColumnSchema.setColumn_group_id(wrapperColumnSchema.getColumnGroupId()); thriftColumnSchema.setScale(wrapperColumnSchema.getScale()); thriftColumnSchema.setPrecision(wrapperColumnSchema.getPrecision()); @@ -359,65 +297,6 @@ private DataType fromExternalToWrapperDataType(org.carbondata.format.DataType da } } - /** - * convert from external to wrapper converter type - * - * @param convertedType - * @return - */ - private ConvertedType fromExternalToWrapperConvertedType( - org.carbondata.format.ConvertedType convertedType) { - if (null == convertedType) { - return null; - } - switch (convertedType) { - case UTF8: - return ConvertedType.UTF8; - case MAP: - return ConvertedType.MAP; - case MAP_KEY_VALUE: - return ConvertedType.MAP_KEY_VALUE; - case LIST: - return ConvertedType.LIST; - case ENUM: - return ConvertedType.ENUM; - case DECIMAL: - return ConvertedType.DECIMAL; - case DATE: - return ConvertedType.DATE; - case TIME_MILLIS: - return ConvertedType.TIME_MILLIS; - case TIMESTAMP_MILLIS: - return ConvertedType.TIMESTAMP_MILLIS; - case RESERVED: - return ConvertedType.RESERVED; - case UINT_8: - return ConvertedType.UINT_8; - case UINT_16: - return ConvertedType.UINT_16; - case UINT_32: - return ConvertedType.UINT_32; - case UINT_64: - return ConvertedType.UINT_64; - case INT_8: - return ConvertedType.INT_8; - case INT_16: - return ConvertedType.INT_16; - case INT_32: - return ConvertedType.INT_32; - case INT_64: - return ConvertedType.INT_64; - case JSON: - return ConvertedType.JSON; - case BSON: - return ConvertedType.BSON; - case INTERVAL: - return ConvertedType.INTERVAL; - default: - return ConvertedType.UTF8; - } - } - /* (non-Javadoc) * convert from external to wrapper columnschema */ @@ -434,8 +313,6 @@ private ConvertedType fromExternalToWrapperConvertedType( encoders.add(fromExternalToWrapperEncoding(encoder)); } wrapperColumnSchema.setEncodingList(encoders); - wrapperColumnSchema.setConvertedType( - fromExternalToWrapperConvertedType(externalColumnSchema.getConverted_type())); wrapperColumnSchema.setNumberOfChild(externalColumnSchema.getNum_child()); wrapperColumnSchema.setPrecision(externalColumnSchema.getPrecision()); wrapperColumnSchema.setColumnGroup(externalColumnSchema.getColumn_group_id()); diff --git a/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/column/CarbonColumn.java b/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/column/CarbonColumn.java index 486ae8e9783..331c6610979 100644 --- a/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/column/CarbonColumn.java +++ b/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/column/CarbonColumn.java @@ -22,7 +22,6 @@ import java.io.Serializable; import java.util.List; -import org.carbondata.core.carbon.metadata.datatype.ConvertedType; import org.carbondata.core.carbon.metadata.datatype.DataType; import org.carbondata.core.carbon.metadata.encoder.Encoding; @@ -54,13 +53,6 @@ public CarbonColumn(ColumnSchema columnSchema, int ordinal) { this.ordinal = ordinal; } - /** - * @return convertedType - */ - public ConvertedType getConvertedType() { - return columnSchema.getConvertedType(); - } - /** * @return columnar or row based */ diff --git a/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/column/ColumnSchema.java b/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/column/ColumnSchema.java index 632f116f567..4d752f654cd 100644 --- a/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/column/ColumnSchema.java +++ b/core/src/main/java/org/carbondata/core/carbon/metadata/schema/table/column/ColumnSchema.java @@ -21,7 +21,6 @@ import java.io.Serializable; import java.util.List; -import org.carbondata.core.carbon.metadata.datatype.ConvertedType; import org.carbondata.core.carbon.metadata.datatype.DataType; import org.carbondata.core.carbon.metadata.encoder.Encoding; @@ -75,12 +74,6 @@ public class ColumnSchema implements Serializable { */ private int columnGroupId = -1; - /** - * Optional When the schema is the result of a conversion from another model - * Used to record the original type to help with cross conversion. - */ - private ConvertedType convertedType; - /** * Used when this column contains decimal data. */ @@ -176,20 +169,6 @@ public void setColumnGroup(int columnGroupId) { this.columnGroupId = columnGroupId; } - /** - * @return the convertedType - */ - public ConvertedType getConvertedType() { - return convertedType; - } - - /** - * @param convertedType the convertedType to set - */ - public void setConvertedType(ConvertedType convertedType) { - this.convertedType = convertedType; - } - /** * @return the scale */ diff --git a/core/src/main/java/org/carbondata/query/util/DataFileFooterConverter.java b/core/src/main/java/org/carbondata/query/util/DataFileFooterConverter.java index ccafe06f2db..bf6121539f4 100644 --- a/core/src/main/java/org/carbondata/query/util/DataFileFooterConverter.java +++ b/core/src/main/java/org/carbondata/query/util/DataFileFooterConverter.java @@ -40,7 +40,6 @@ import org.carbondata.core.carbon.metadata.blocklet.index.BlockletIndex; import org.carbondata.core.carbon.metadata.blocklet.index.BlockletMinMaxIndex; import org.carbondata.core.carbon.metadata.blocklet.sort.SortState; -import org.carbondata.core.carbon.metadata.datatype.ConvertedType; import org.carbondata.core.carbon.metadata.datatype.DataType; import org.carbondata.core.carbon.metadata.encoder.Encoding; import org.carbondata.core.carbon.metadata.schema.table.column.ColumnSchema; @@ -162,8 +161,6 @@ private ColumnSchema thriftColumnSchmeaToWrapperColumnSchema( encoders.add(fromExternalToWrapperEncoding(encoder)); } wrapperColumnSchema.setEncodingList(encoders); - wrapperColumnSchema.setConvertedType(thriftConvertedTypeToWrapperConvertedTypeConverter( - externalColumnSchema.getConverted_type())); wrapperColumnSchema.setNumberOfChild(externalColumnSchema.getNum_child()); wrapperColumnSchema.setPrecision(externalColumnSchema.getPrecision()); wrapperColumnSchema.setColumnGroup(externalColumnSchema.getColumn_group_id()); @@ -323,66 +320,6 @@ private DataType thriftDataTyopeToWrapperDataType(org.carbondata.format.DataType } } - /** - * Below method is to convert the thrift converted type to wrapper converted - * type - * - * @param convertedType thrift - * @return wrapper converted type - */ - private ConvertedType thriftConvertedTypeToWrapperConvertedTypeConverter( - org.carbondata.format.ConvertedType convertedType) { - if (null == convertedType) { - return null; - } - switch (convertedType) { - case UTF8: - return ConvertedType.UTF8; - case MAP: - return ConvertedType.MAP; - case MAP_KEY_VALUE: - return ConvertedType.MAP_KEY_VALUE; - case LIST: - return ConvertedType.LIST; - case ENUM: - return ConvertedType.ENUM; - case DECIMAL: - return ConvertedType.DECIMAL; - case DATE: - return ConvertedType.DATE; - case TIME_MILLIS: - return ConvertedType.TIME_MILLIS; - case TIMESTAMP_MILLIS: - return ConvertedType.TIMESTAMP_MILLIS; - case RESERVED: - return ConvertedType.RESERVED; - case UINT_8: - return ConvertedType.UINT_8; - case UINT_16: - return ConvertedType.UINT_16; - case UINT_32: - return ConvertedType.UINT_32; - case UINT_64: - return ConvertedType.UINT_64; - case INT_8: - return ConvertedType.INT_8; - case INT_16: - return ConvertedType.INT_16; - case INT_32: - return ConvertedType.INT_32; - case INT_64: - return ConvertedType.INT_64; - case JSON: - return ConvertedType.JSON; - case BSON: - return ConvertedType.BSON; - case INTERVAL: - return ConvertedType.INTERVAL; - default: - return ConvertedType.UTF8; - } - } - /** * Below method will be used to convert the thrift presence meta to wrapper * presence meta diff --git a/format/src/main/thrift/schema.thrift b/format/src/main/thrift/schema.thrift index 96905b8f180..96444ba4b13 100644 --- a/format/src/main/thrift/schema.thrift +++ b/format/src/main/thrift/schema.thrift @@ -36,128 +36,6 @@ enum DataType { STRUCT = 21, } -/** - * Common types used by frameworks(e.g. hive, pig, spark) using carbondata. This helps map - * between types in those frameworks to the base types in carbondata. This is only - * metadata and not needed to read or write the data. - */ -enum ConvertedType { - /** a BYTE_ARRAY actually contains UTF8 encoded chars */ - UTF8 = 0, - - /** a map is converted as an optional field containing a repeated key/value pair */ - MAP = 1, - - /** a key/value pair is converted into a group of two fields */ - MAP_KEY_VALUE = 2, - - /** a list is converted into an optional field containing a repeated field for its - * values */ - LIST = 3, - - /** an enum is converted into a binary field */ - ENUM = 4, - - /** - * A decimal value. - * - * This may be used to annotate binary or fixed primitive types. The - * underlying byte array stores the unscaled value encoded as two's - * complement using big-endian byte order (the most significant byte is the - * zeroth element). The value of the decimal is the value * 10^{-scale}. - * - * This must be accompanied by a (maximum) precision and a scale in the - * SchemaElement. The precision specifies the number of digits in the decimal - * and the scale stores the location of the decimal point. For example 1.23 - * would have precision 3 (3 total digits) and scale 2 (the decimal point is - * 2 digits over). - */ - DECIMAL = 5, - - /** - * A Date - * - * Stored as days since Unix epoch, encoded as the INT32 physical type. - * - */ - DATE = 6, - - /** - * A time - * - * The total number of milliseconds since midnight. The value is stored - * as an INT32 physical type. - */ - TIME_MILLIS = 7, - - /** - * A date/time combination - * - * Date and time recorded as milliseconds since the Unix epoch. Recorded as - * a physical type of INT64. - */ - TIMESTAMP_MILLIS = 8, - - RESERVED = 10, - - /** - * An unsigned integer value. - * - * The number describes the maximum number of meainful data bits in - * the stored value. 8, 16 and 32 bit values are stored using the - * INT32 physical type. 64 bit values are stored using the INT64 - * physical type. - * - */ - UINT_8 = 11, - UINT_16 = 12, - UINT_32 = 13, - UINT_64 = 14, - - /** - * A signed integer value. - * - * The number describes the maximum number of meainful data bits in - * the stored value. 8, 16 and 32 bit values are stored using the - * INT32 physical type. 64 bit values are stored using the INT64 - * physical type. - * - */ - INT_8 = 15, - INT_16 = 16, - INT_32 = 17, - INT_64 = 18, - - /** - * An embedded JSON document - * - * A JSON document embedded within a single UTF8 column. - */ - JSON = 19, - - /** - * An embedded BSON document - * - * A BSON document embedded within a single BINARY column. - */ - BSON = 20, - - /** - * An interval of time - * - * This type annotates data stored as a FIXED_LEN_BYTE_ARRAY of length 12 - * This data is composed of three separate little endian unsigned - * integers. Each stores a component of a duration of time. The first - * integer identifies the number of months associated with the duration, - * the second identifies the number of days associated with the duration - * and the third identifies the number of milliseconds associated with - * the provided duration. This duration of time is independent of any - * particular timezone or date. - */ - INTERVAL = 21 - -} - /** * Encodings supported by Carbon Data. Not all encodings are valid for all types. * Certain Encodings can be chained. @@ -188,30 +66,25 @@ struct ColumnSchema{ 5: required list encoders; // List of encoders that are chained to encode the data for this column 6: required bool dimension; // Whether the column is a dimension or measure 7: optional i32 column_group_id; // The group ID for column used for row format columns, where in columns in each group are chunked together. - - /** When the schema is the result of a conversion from another model - * Used to record the original type to help with cross conversion. - */ - 8: optional ConvertedType converted_type; - /** + /** * Used when this column contains decimal data. */ - 9: optional i32 scale; - 10: optional i32 precision; + 8: optional i32 scale; + 9: optional i32 precision; /** Nested fields. Since thrift does not support nested fields, * the nesting is flattened to a single list by a depth-first traversal. * The children count is used to construct the nested relationship. * This field is not set when the element is a primitive type */ - 11: optional i32 num_child; + 10: optional i32 num_child; /** * Used when this column is part of an aggregate table. */ - 12: optional string aggregate_function; + 11: optional string aggregate_function; - 13: optional binary default_value; + 12: optional binary default_value; } /**