diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java index b1d3647a545..9cd14272d85 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/data/manager/BaseTableDataManager.java @@ -349,7 +349,6 @@ public Pair fetchTableConfigAndSchema() { public IndexLoadingConfig getIndexLoadingConfig(TableConfig tableConfig, @Nullable Schema schema) { IndexLoadingConfig indexLoadingConfig = new IndexLoadingConfig(_instanceDataManagerConfig, tableConfig, schema); indexLoadingConfig.setTableDataDir(_tableDataDir); - indexLoadingConfig.setInstanceTierConfigs(_instanceDataManagerConfig.getTierConfigs()); return indexLoadingConfig; } @@ -620,7 +619,6 @@ public void reloadSegment(String segmentName, IndexLoadingConfig indexLoadingCon String segmentTier = getSegmentCurrentTier(segmentName); indexLoadingConfig.setSegmentTier(segmentTier); indexLoadingConfig.setTableDataDir(_tableDataDir); - indexLoadingConfig.setInstanceTierConfigs(_instanceDataManagerConfig.getTierConfigs()); File indexDir = getSegmentDataDir(segmentName, segmentTier, indexLoadingConfig.getTableConfig()); Lock segmentLock = getSegmentLock(segmentName); segmentLock.lock(); diff --git a/pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerTest.java b/pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerTest.java index 7d351c486f6..69a8d88fd69 100644 --- a/pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerTest.java +++ b/pinot-core/src/test/java/org/apache/pinot/core/data/manager/BaseTableDataManagerTest.java @@ -720,7 +720,6 @@ private IndexLoadingConfig createTierIndexLoadingConfig(TableConfig tableConfig) when(instanceDataManagerConfig.getConfig()).thenReturn(new PinotConfiguration()); IndexLoadingConfig indexLoadingConfig = new IndexLoadingConfig(instanceDataManagerConfig, tableConfig, null); indexLoadingConfig.setTableDataDir(TEMP_DIR.getAbsolutePath() + File.separator + tableConfig.getTableName()); - indexLoadingConfig.setInstanceTierConfigs(Map.of()); indexLoadingConfig.setSegmentTier(TIER_NAME); return indexLoadingConfig; } diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/bloom/BloomIndexType.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/bloom/BloomIndexType.java index 84bac9ccdcc..e6e2818c04b 100644 --- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/bloom/BloomIndexType.java +++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/bloom/BloomIndexType.java @@ -24,8 +24,6 @@ import java.util.Map; import javax.annotation.Nullable; import org.apache.pinot.segment.local.segment.creator.impl.bloom.OnHeapGuavaBloomFilterCreator; -import org.apache.pinot.segment.local.segment.index.loader.ConfigurableFromIndexLoadingConfig; -import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig; import org.apache.pinot.segment.local.segment.index.loader.bloomfilter.BloomFilterHandler; import org.apache.pinot.segment.local.segment.index.readers.bloom.BloomFilterReaderFactory; import org.apache.pinot.segment.spi.ColumnMetadata; @@ -49,9 +47,7 @@ import org.apache.pinot.spi.data.Schema; -public class BloomIndexType - extends AbstractIndexType - implements ConfigurableFromIndexLoadingConfig { +public class BloomIndexType extends AbstractIndexType { public static final String INDEX_DISPLAY_NAME = "bloom"; private static final List EXTENSIONS = Collections.singletonList(V1Constants.Indexes.BLOOM_FILTER_FILE_EXTENSION); @@ -65,11 +61,6 @@ public Class getIndexConfigClass() { return BloomFilterConfig.class; } - @Override - public Map fromIndexLoadingConfig(IndexLoadingConfig indexLoadingConfig) { - return indexLoadingConfig.getBloomFilterConfigs(); - } - @Override public BloomFilterConfig getDefaultConfig() { return BloomFilterConfig.DISABLED; @@ -82,16 +73,14 @@ public String getPrettyName() { @Override public ColumnConfigDeserializer createDeserializer() { - return IndexConfigDeserializer.fromIndexes(getPrettyName(), getIndexConfigClass()) - .withExclusiveAlternative( - IndexConfigDeserializer.ifIndexingConfig(// reads tableConfig.indexingConfig.bloomFilterConfigs - IndexConfigDeserializer.fromMap(tableConfig -> tableConfig.getIndexingConfig().getBloomFilterConfigs()) - .withFallbackAlternative(// reads tableConfig.indexingConfig.bloomFilterColumns - IndexConfigDeserializer.fromCollection( - tableConfig -> tableConfig.getIndexingConfig().getBloomFilterColumns(), - (accum, column) -> accum.put(column, BloomFilterConfig.DEFAULT))) - ) - ); + ColumnConfigDeserializer fromIndexes = + IndexConfigDeserializer.fromIndexes(getPrettyName(), getIndexConfigClass()); + ColumnConfigDeserializer fromBloomFilterConfigs = + IndexConfigDeserializer.fromMap(tableConfig -> tableConfig.getIndexingConfig().getBloomFilterConfigs()); + ColumnConfigDeserializer fromBloomFilterColumns = + IndexConfigDeserializer.fromCollection(tableConfig -> tableConfig.getIndexingConfig().getBloomFilterColumns(), + (accum, column) -> accum.put(column, BloomFilterConfig.DEFAULT)); + return fromIndexes.withExclusiveAlternative(fromBloomFilterConfigs.withFallbackAlternative(fromBloomFilterColumns)); } @Override diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/dictionary/DictionaryIndexType.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/dictionary/DictionaryIndexType.java index 9fb6dbce87c..6d668da2b2e 100644 --- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/dictionary/DictionaryIndexType.java +++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/dictionary/DictionaryIndexType.java @@ -26,7 +26,6 @@ import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -37,8 +36,6 @@ import org.apache.pinot.segment.local.io.util.PinotDataBitSet; import org.apache.pinot.segment.local.realtime.impl.dictionary.MutableDictionaryFactory; import org.apache.pinot.segment.local.segment.creator.impl.SegmentDictionaryCreator; -import org.apache.pinot.segment.local.segment.index.loader.ConfigurableFromIndexLoadingConfig; -import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig; import org.apache.pinot.segment.local.segment.index.readers.BigDecimalDictionary; import org.apache.pinot.segment.local.segment.index.readers.BytesDictionary; import org.apache.pinot.segment.local.segment.index.readers.DoubleDictionary; @@ -88,8 +85,7 @@ public class DictionaryIndexType - extends AbstractIndexType - implements ConfigurableFromIndexLoadingConfig { + extends AbstractIndexType { private static final Logger LOGGER = LoggerFactory.getLogger(DictionaryIndexType.class); private static final List EXTENSIONS = Collections.singletonList(V1Constants.Dict.FILE_EXTENSION); @@ -102,24 +98,6 @@ public Class getIndexConfigClass() { return DictionaryIndexConfig.class; } - @Override - public Map fromIndexLoadingConfig( - IndexLoadingConfig indexLoadingConfig) { - Map result = new HashMap<>(); - Set noDictionaryCols = indexLoadingConfig.getNoDictionaryColumns(); - Set onHeapCols = indexLoadingConfig.getOnHeapDictionaryColumns(); - Set varLengthCols = indexLoadingConfig.getVarLengthDictionaryColumns(); - for (String column : indexLoadingConfig.getAllKnownColumns()) { - if (noDictionaryCols.contains(column)) { - result.put(column, DictionaryIndexConfig.disabled()); - } else { - // Intern configs can only be used if dictionary is enabled through FieldConfigLists. - result.put(column, new DictionaryIndexConfig(onHeapCols.contains(column), varLengthCols.contains(column))); - } - } - return result; - } - @Override public DictionaryIndexConfig getDefaultConfig() { return DictionaryIndexConfig.DEFAULT; diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/fst/FstIndexType.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/fst/FstIndexType.java index 83e755f734e..72f75ad097d 100644 --- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/fst/FstIndexType.java +++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/fst/FstIndexType.java @@ -22,14 +22,10 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import java.io.IOException; -import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import javax.annotation.Nullable; import org.apache.pinot.segment.local.segment.creator.impl.inv.text.LuceneFSTIndexCreator; -import org.apache.pinot.segment.local.segment.index.loader.ConfigurableFromIndexLoadingConfig; -import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig; import org.apache.pinot.segment.local.segment.index.loader.invertedindex.FSTIndexHandler; import org.apache.pinot.segment.local.segment.index.readers.LuceneFSTIndexReader; import org.apache.pinot.segment.local.utils.nativefst.FSTHeader; @@ -60,8 +56,7 @@ import org.apache.pinot.spi.data.Schema; -public class FstIndexType extends AbstractIndexType - implements ConfigurableFromIndexLoadingConfig { +public class FstIndexType extends AbstractIndexType { public static final String INDEX_DISPLAY_NAME = "fst"; private static final List EXTENSIONS = ImmutableList.of(V1Constants.Indexes.LUCENE_FST_INDEX_FILE_EXTENSION, @@ -77,51 +72,6 @@ public Class getIndexConfigClass() { return FstIndexConfig.class; } - @Override - public Map fromIndexLoadingConfig(IndexLoadingConfig indexLoadingConfig) { - Map result = new HashMap<>(); - Set fstIndexColumns = indexLoadingConfig.getFSTIndexColumns(); - for (String column : indexLoadingConfig.getAllKnownColumns()) { - if (fstIndexColumns.contains(column)) { - FSTType fstType = getFstTypeFromIndexLoadingConfig(indexLoadingConfig, column); - FstIndexConfig conf = new FstIndexConfig(fstType); - result.put(column, conf); - } else { - result.put(column, FstIndexConfig.DISABLED); - } - } - return result; - } - - private FSTType getFstTypeFromIndexLoadingConfig(IndexLoadingConfig indexLoadingConfig, String column) { - - FSTType fstType = indexLoadingConfig.getFSTIndexType(); - - TableConfig tableConfig = indexLoadingConfig.getTableConfig(); - if (tableConfig != null) { - List fieldConfigList = tableConfig.getFieldConfigList(); - if (fieldConfigList != null) { - FieldConfig fieldConfig = fieldConfigList.stream() - .filter(fc -> fc.getName().equals(column)) - .findAny() - .orElse(null); - if (fieldConfig != null) { - Map textProperties = fieldConfig.getProperties(); - if (textProperties != null) { - for (Map.Entry entry : textProperties.entrySet()) { - if (entry.getKey().equalsIgnoreCase(FieldConfig.TEXT_FST_TYPE)) { - fstType = FSTType.NATIVE; - } else { - fstType = FSTType.LUCENE; - } - } - } - } - } - } - return fstType; - } - @Override public FstIndexConfig getDefaultConfig() { return FstIndexConfig.DISABLED; diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/inverted/InvertedIndexType.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/inverted/InvertedIndexType.java index b3ca31ffbb9..4b5fcc8daec 100644 --- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/inverted/InvertedIndexType.java +++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/inverted/InvertedIndexType.java @@ -24,14 +24,10 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; import javax.annotation.Nullable; import org.apache.pinot.segment.local.realtime.impl.invertedindex.RealtimeInvertedIndex; import org.apache.pinot.segment.local.segment.creator.impl.inv.OffHeapBitmapInvertedIndexCreator; import org.apache.pinot.segment.local.segment.creator.impl.inv.OnHeapBitmapInvertedIndexCreator; -import org.apache.pinot.segment.local.segment.index.loader.ConfigurableFromIndexLoadingConfig; -import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig; import org.apache.pinot.segment.local.segment.index.loader.invertedindex.InvertedIndexHandler; import org.apache.pinot.segment.local.segment.index.readers.BitmapInvertedIndexReader; import org.apache.pinot.segment.spi.ColumnMetadata; @@ -60,8 +56,7 @@ public class InvertedIndexType - extends AbstractIndexType - implements ConfigurableFromIndexLoadingConfig { + extends AbstractIndexType { public static final String INDEX_DISPLAY_NAME = "inverted"; private static final List EXTENSIONS = Collections.singletonList(V1Constants.Indexes.BITMAP_INVERTED_INDEX_FILE_EXTENSION); @@ -75,12 +70,6 @@ public Class getIndexConfigClass() { return IndexConfig.class; } - @Override - public Map fromIndexLoadingConfig(IndexLoadingConfig indexLoadingConfig) { - return indexLoadingConfig.getInvertedIndexColumns().stream() - .collect(Collectors.toMap(Function.identity(), v -> IndexConfig.ENABLED)); - } - @Override public IndexConfig getDefaultConfig() { return IndexConfig.DISABLED; @@ -93,11 +82,12 @@ public String getPrettyName() { @Override public ColumnConfigDeserializer createDeserializer() { - ColumnConfigDeserializer fromInvertedCols = IndexConfigDeserializer.fromCollection( - tableConfig -> tableConfig.getIndexingConfig().getInvertedIndexColumns(), - (acum, column) -> acum.put(column, IndexConfig.ENABLED)); - return IndexConfigDeserializer.fromIndexes(getPrettyName(), getIndexConfigClass()) - .withExclusiveAlternative(IndexConfigDeserializer.ifIndexingConfig(fromInvertedCols)); + ColumnConfigDeserializer fromIndexes = + IndexConfigDeserializer.fromIndexes(getPrettyName(), getIndexConfigClass()); + ColumnConfigDeserializer fromInvertedIndexColumns = + IndexConfigDeserializer.fromCollection(tableConfig -> tableConfig.getIndexingConfig().getInvertedIndexColumns(), + (acum, column) -> acum.put(column, IndexConfig.ENABLED)); + return fromIndexes.withExclusiveAlternative(fromInvertedIndexColumns); } public DictionaryBasedInvertedIndexCreator createIndexCreator(IndexCreationContext context) @@ -112,8 +102,7 @@ public DictionaryBasedInvertedIndexCreator createIndexCreator(IndexCreationConte } @Override - public DictionaryBasedInvertedIndexCreator createIndexCreator(IndexCreationContext context, - IndexConfig indexConfig) + public DictionaryBasedInvertedIndexCreator createIndexCreator(IndexCreationContext context, IndexConfig indexConfig) throws IOException { return createIndexCreator(context); } @@ -170,8 +159,8 @@ public InvertedIndexReader createIndexReader(SegmentDirectory.Reader segmentRead + "index if it has no dictionary"); } if (metadata.isSorted() && metadata.isSingleValue()) { - ForwardIndexReader fwdReader = StandardIndexes.forward().getReaderFactory() - .createIndexReader(segmentReader, fieldIndexConfigs, metadata); + ForwardIndexReader fwdReader = + StandardIndexes.forward().getReaderFactory().createIndexReader(segmentReader, fieldIndexConfigs, metadata); Preconditions.checkState(fwdReader instanceof SortedIndexReader); return (SortedIndexReader) fwdReader; } else { diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/json/JsonIndexType.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/json/JsonIndexType.java index 6206842e9dc..9441cbd40b5 100644 --- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/json/JsonIndexType.java +++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/json/JsonIndexType.java @@ -28,8 +28,6 @@ import org.apache.pinot.segment.local.realtime.impl.json.MutableJsonIndexImpl; import org.apache.pinot.segment.local.segment.creator.impl.inv.json.OffHeapJsonIndexCreator; import org.apache.pinot.segment.local.segment.creator.impl.inv.json.OnHeapJsonIndexCreator; -import org.apache.pinot.segment.local.segment.index.loader.ConfigurableFromIndexLoadingConfig; -import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig; import org.apache.pinot.segment.local.segment.index.loader.invertedindex.JsonIndexHandler; import org.apache.pinot.segment.local.segment.index.readers.json.ImmutableJsonIndexReader; import org.apache.pinot.segment.spi.ColumnMetadata; @@ -56,8 +54,7 @@ import org.apache.pinot.spi.data.Schema; -public class JsonIndexType extends AbstractIndexType - implements ConfigurableFromIndexLoadingConfig { +public class JsonIndexType extends AbstractIndexType { public static final String INDEX_DISPLAY_NAME = "json"; private static final List EXTENSIONS = Collections.singletonList(V1Constants.Indexes.JSON_INDEX_FILE_EXTENSION); @@ -71,11 +68,6 @@ public Class getIndexConfigClass() { return JsonIndexConfig.class; } - @Override - public Map fromIndexLoadingConfig(IndexLoadingConfig indexLoadingConfig) { - return indexLoadingConfig.getJsonIndexConfigs(); - } - @Override public JsonIndexConfig getDefaultConfig() { return JsonIndexConfig.DISABLED; @@ -88,17 +80,14 @@ public String getPrettyName() { @Override public ColumnConfigDeserializer createDeserializer() { - // reads tableConfig.indexingConfig.jsonIndexConfigs - ColumnConfigDeserializer fromJsonIndexConf = + ColumnConfigDeserializer fromIndexes = + IndexConfigDeserializer.fromIndexes(getPrettyName(), getIndexConfigClass()); + ColumnConfigDeserializer fromJsonIndexConfigs = IndexConfigDeserializer.fromMap(tableConfig -> tableConfig.getIndexingConfig().getJsonIndexConfigs()); - // reads tableConfig.indexingConfig.jsonIndexColumns - ColumnConfigDeserializer fromJsonIndexCols = - IndexConfigDeserializer.fromCollection( - tableConfig -> tableConfig.getIndexingConfig().getJsonIndexColumns(), - (accum, column) -> accum.put(column, new JsonIndexConfig())); - return IndexConfigDeserializer.fromIndexes(getPrettyName(), getIndexConfigClass()) - .withExclusiveAlternative( - IndexConfigDeserializer.ifIndexingConfig(fromJsonIndexCols.withExclusiveAlternative(fromJsonIndexConf))); + ColumnConfigDeserializer fromJsonIndexColumns = + IndexConfigDeserializer.fromCollection(tableConfig -> tableConfig.getIndexingConfig().getJsonIndexColumns(), + (accum, column) -> accum.put(column, JsonIndexConfig.DEFAULT)); + return fromIndexes.withExclusiveAlternative(fromJsonIndexConfigs.withFallbackAlternative(fromJsonIndexColumns)); } @Override @@ -108,8 +97,8 @@ public JsonIndexCreator createIndexCreator(IndexCreationContext context, JsonInd "Json index is currently only supported on single-value columns"); Preconditions.checkState(context.getFieldSpec().getDataType().getStoredType() == FieldSpec.DataType.STRING, "Json index is currently only supported on STRING columns"); - return context.isOnHeap() - ? new OnHeapJsonIndexCreator(context.getIndexDir(), context.getFieldSpec().getName(), indexConfig) + return context.isOnHeap() ? new OnHeapJsonIndexCreator(context.getIndexDir(), context.getFieldSpec().getName(), + indexConfig) : new OffHeapJsonIndexCreator(context.getIndexDir(), context.getFieldSpec().getName(), indexConfig); } diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/ConfigurableFromIndexLoadingConfig.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/ConfigurableFromIndexLoadingConfig.java deleted file mode 100644 index 2fe78011a4d..00000000000 --- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/ConfigurableFromIndexLoadingConfig.java +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.pinot.segment.local.segment.index.loader; - -import java.util.Map; -import org.apache.pinot.spi.config.table.IndexConfig; - - -/** - * This interface can be optionally implemented by {@link org.apache.pinot.segment.spi.index.IndexType index types} to - * indicate that they can extract their configuration from an older {@link IndexLoadingConfig} object. - */ -public interface ConfigurableFromIndexLoadingConfig { - - - /** - * Returns a map that can be used to get the index config. - * - * This map is used with higher priority whenever the index configuration needs to be read from an - * {@link IndexLoadingConfig}. - * - * Sometimes {@link IndexLoadingConfig} is not completely configured and - * {@link IndexLoadingConfig#getAllKnownColumns()} does not return all columns in the table. - * Therefore the returned map may not have an entry for each column in the actual schema. - * - * @return a map whose keys are the column names and the values are the index configuration for that column. - */ - Map fromIndexLoadingConfig(IndexLoadingConfig indexLoadingConfig); -} diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/IndexLoadingConfig.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/IndexLoadingConfig.java index 51e33e54445..84094bfd0b5 100644 --- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/IndexLoadingConfig.java +++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/loader/IndexLoadingConfig.java @@ -19,33 +19,26 @@ package org.apache.pinot.segment.local.segment.index.loader; import com.google.common.annotations.VisibleForTesting; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; import javax.annotation.Nullable; import org.apache.commons.lang3.StringUtils; import org.apache.pinot.common.utils.config.TableConfigUtils; import org.apache.pinot.segment.local.segment.index.column.PhysicalColumnIndexContainer; import org.apache.pinot.segment.local.segment.index.loader.columnminmaxvalue.ColumnMinMaxValueGeneratorMode; import org.apache.pinot.segment.spi.creator.SegmentVersion; -import org.apache.pinot.segment.spi.index.ColumnConfigDeserializer; import org.apache.pinot.segment.spi.index.FieldIndexConfigs; import org.apache.pinot.segment.spi.index.FieldIndexConfigsUtil; -import org.apache.pinot.segment.spi.index.IndexConfigDeserializer; -import org.apache.pinot.segment.spi.index.IndexType; import org.apache.pinot.segment.spi.index.RangeIndexConfig; import org.apache.pinot.segment.spi.loader.SegmentDirectoryLoaderRegistry; import org.apache.pinot.spi.config.instance.InstanceDataManagerConfig; import org.apache.pinot.spi.config.table.BloomFilterConfig; import org.apache.pinot.spi.config.table.FSTType; import org.apache.pinot.spi.config.table.FieldConfig; -import org.apache.pinot.spi.config.table.IndexConfig; import org.apache.pinot.spi.config.table.IndexingConfig; import org.apache.pinot.spi.config.table.JsonIndexConfig; import org.apache.pinot.spi.config.table.StarTreeIndexConfig; @@ -57,8 +50,6 @@ import org.apache.pinot.spi.utils.CommonConstants; import org.apache.pinot.spi.utils.ReadMode; import org.apache.pinot.spi.utils.TimestampIndexUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** @@ -67,58 +58,63 @@ public class IndexLoadingConfig { private static final int DEFAULT_REALTIME_AVG_MULTI_VALUE_COUNT = 2; public static final String READ_MODE_KEY = "readMode"; - private static final Logger LOGGER = LoggerFactory.getLogger(IndexLoadingConfig.class); - private InstanceDataManagerConfig _instanceDataManagerConfig = null; + private final InstanceDataManagerConfig _instanceDataManagerConfig; + private final TableConfig _tableConfig; + private final Schema _schema; + + // These fields can be modified after initialization + // TODO: Revisit them private ReadMode _readMode = ReadMode.DEFAULT_MODE; + private SegmentVersion _segmentVersion; + private String _segmentTier; + private Set _knownColumns; + private String _tableDataDir; + private boolean _errorOnColumnBuildFailure; + + // Initialized by instance data manager config + private String _instanceId; + private boolean _isRealtimeOffHeapAllocation; + private boolean _isDirectRealtimeOffHeapAllocation; + private int _realtimeAvgMultiValueCount = DEFAULT_REALTIME_AVG_MULTI_VALUE_COUNT; + private String _segmentStoreURI; + private String _segmentDirectoryLoader; + private Map> _instanceTierConfigs; + + // Initialized by table config and schema private List _sortedColumns = Collections.emptyList(); - private Set _invertedIndexColumns = new HashSet<>(); - private Set _rangeIndexColumns = new HashSet<>(); + private final Set _invertedIndexColumns = new HashSet<>(); + private final Set _rangeIndexColumns = new HashSet<>(); private int _rangeIndexVersion = RangeIndexConfig.DEFAULT.getVersion(); - private Set _textIndexColumns = new HashSet<>(); - private Set _fstIndexColumns = new HashSet<>(); + private final Set _textIndexColumns = new HashSet<>(); + private final Set _fstIndexColumns = new HashSet<>(); private FSTType _fstIndexType = FSTType.LUCENE; private Map _jsonIndexConfigs = new HashMap<>(); private final Set _noDictionaryColumns = new HashSet<>(); // TODO: replace this by _noDictionaryConfig. private final Map _noDictionaryConfig = new HashMap<>(); private final Set _varLengthDictionaryColumns = new HashSet<>(); - private Set _onHeapDictionaryColumns = new HashSet<>(); - private Map _bloomFilterConfigs = new HashMap<>(); + private final Set _onHeapDictionaryColumns = new HashSet<>(); + private final Map _bloomFilterConfigs = new HashMap<>(); + private ColumnMinMaxValueGeneratorMode _columnMinMaxValueGeneratorMode = ColumnMinMaxValueGeneratorMode.DEFAULT_MODE; private boolean _enableDynamicStarTreeCreation; private List _starTreeIndexConfigs; private boolean _enableDefaultStarTree; + private final Map> _columnProperties = new HashMap<>(); private Map _indexConfigsByColName = new HashMap<>(); - private SegmentVersion _segmentVersion; - private ColumnMinMaxValueGeneratorMode _columnMinMaxValueGeneratorMode = ColumnMinMaxValueGeneratorMode.DEFAULT_MODE; - private int _realtimeAvgMultiValueCount = DEFAULT_REALTIME_AVG_MULTI_VALUE_COUNT; - private boolean _isRealtimeOffHeapAllocation; - private boolean _isDirectRealtimeOffHeapAllocation; - private String _segmentStoreURI; - private boolean _errorOnColumnBuildFailure; - - // constructed from FieldConfig - private Map> _columnProperties = new HashMap<>(); - - @Nullable - private TableConfig _tableConfig; - private Schema _schema; - private String _tableDataDir; - private String _segmentDirectoryLoader; - private String _segmentTier; - - private String _instanceId; - private Map> _instanceTierConfigs; private boolean _dirty = true; - private Set _knownColumns = null; /** * NOTE: This step might modify the passed in table config and schema. + * + * TODO: Revisit the init handling. Currently it doesn't apply tiered config override */ - public IndexLoadingConfig(InstanceDataManagerConfig instanceDataManagerConfig, TableConfig tableConfig, - @Nullable Schema schema) { - extractFromInstanceConfig(instanceDataManagerConfig); - extractFromTableConfigAndSchema(tableConfig, schema); + public IndexLoadingConfig(@Nullable InstanceDataManagerConfig instanceDataManagerConfig, + @Nullable TableConfig tableConfig, @Nullable Schema schema) { + _instanceDataManagerConfig = instanceDataManagerConfig; + _tableConfig = tableConfig; + _schema = schema; + init(); } @VisibleForTesting @@ -128,27 +124,74 @@ public IndexLoadingConfig(InstanceDataManagerConfig instanceDataManagerConfig, T @VisibleForTesting public IndexLoadingConfig(TableConfig tableConfig, @Nullable Schema schema) { - extractFromTableConfigAndSchema(tableConfig, schema); + this(null, tableConfig, schema); } /** * NOTE: Can be used in production code when we want to load a segment as is without any modifications. */ public IndexLoadingConfig() { + this(null, null, null); } + @Nullable public InstanceDataManagerConfig getInstanceDataManagerConfig() { return _instanceDataManagerConfig; } - private void extractFromTableConfigAndSchema(TableConfig tableConfig, @Nullable Schema schema) { - if (schema != null) { - TimestampIndexUtils.applyTimestampIndex(tableConfig, schema); + @Nullable + public TableConfig getTableConfig() { + return _tableConfig; + } + + @Nullable + public Schema getSchema() { + return _schema; + } + + private void init() { + if (_instanceDataManagerConfig != null) { + extractFromInstanceConfig(); } - _tableConfig = tableConfig; - _schema = schema; + if (_tableConfig != null) { + extractFromTableConfigAndSchema(); + } + } - IndexingConfig indexingConfig = tableConfig.getIndexingConfig(); + private void extractFromInstanceConfig() { + _instanceId = _instanceDataManagerConfig.getInstanceId(); + + ReadMode instanceReadMode = _instanceDataManagerConfig.getReadMode(); + if (instanceReadMode != null) { + _readMode = instanceReadMode; + } + + String instanceSegmentVersion = _instanceDataManagerConfig.getSegmentFormatVersion(); + if (instanceSegmentVersion != null) { + _segmentVersion = SegmentVersion.valueOf(instanceSegmentVersion.toLowerCase()); + } + + _isRealtimeOffHeapAllocation = _instanceDataManagerConfig.isRealtimeOffHeapAllocation(); + _isDirectRealtimeOffHeapAllocation = _instanceDataManagerConfig.isDirectRealtimeOffHeapAllocation(); + + String avgMultiValueCount = _instanceDataManagerConfig.getAvgMultiValueCount(); + if (avgMultiValueCount != null) { + _realtimeAvgMultiValueCount = Integer.parseInt(avgMultiValueCount); + } + _segmentStoreURI = + _instanceDataManagerConfig.getConfig().getProperty(CommonConstants.Server.CONFIG_OF_SEGMENT_STORE_URI); + _segmentDirectoryLoader = _instanceDataManagerConfig.getSegmentDirectoryLoader(); + + Map> tierConfigs = _instanceDataManagerConfig.getTierConfigs(); + _instanceTierConfigs = tierConfigs != null ? tierConfigs : Map.of(); + } + + private void extractFromTableConfigAndSchema() { + if (_schema != null) { + TimestampIndexUtils.applyTimestampIndex(_tableConfig, _schema); + } + + IndexingConfig indexingConfig = _tableConfig.getIndexingConfig(); String tableReadMode = indexingConfig.getLoadMode(); if (tableReadMode != null) { _readMode = ReadMode.getEnum(tableReadMode); @@ -203,16 +246,25 @@ private void extractFromTableConfigAndSchema(TableConfig tableConfig, @Nullable _noDictionaryColumns.addAll(noDictionaryColumns); } - List fieldConfigList = tableConfig.getFieldConfigList(); - if (fieldConfigList != null) { - for (FieldConfig fieldConfig : fieldConfigList) { + List fieldConfigs = _tableConfig.getFieldConfigList(); + if (fieldConfigs != null) { + for (FieldConfig fieldConfig : fieldConfigs) { _columnProperties.put(fieldConfig.getName(), fieldConfig.getProperties()); + List indexTypes = fieldConfig.getIndexTypes(); + for (FieldConfig.IndexType indexType : indexTypes) { + if (indexType == FieldConfig.IndexType.INVERTED) { + _invertedIndexColumns.add(fieldConfig.getName()); + } else if (indexType == FieldConfig.IndexType.RANGE) { + _rangeIndexColumns.add(fieldConfig.getName()); + } else if (indexType == FieldConfig.IndexType.TEXT) { + _textIndexColumns.add(fieldConfig.getName()); + } else if (indexType == FieldConfig.IndexType.FST) { + _fstIndexColumns.add(fieldConfig.getName()); + } + } } } - extractTextIndexColumnsFromTableConfig(tableConfig); - extractFSTIndexColumnsFromTableConfig(tableConfig); - Map noDictionaryConfig = indexingConfig.getNoDictionaryConfig(); if (noDictionaryConfig != null) { _noDictionaryConfig.putAll(noDictionaryConfig); @@ -243,77 +295,24 @@ private void extractFromTableConfigAndSchema(TableConfig tableConfig, @Nullable } public void refreshIndexConfigs() { - TableConfig tableConfig = getTableConfigWithTierOverwrites(); + if (_tableConfig == null) { + _dirty = false; + return; + } // Accessing the index configs for single-column index is handled by IndexType.getConfig() as defined in index-spi. // As the tableConfig is overwritten with tier specific configs, IndexType.getConfig() can access the tier // specific index configs transparently. - _indexConfigsByColName = calculateIndexConfigsByColName(tableConfig, inferSchema()); + TableConfig tableConfig = getTableConfigWithTierOverwrites(); + Schema schema = inferSchema(); + _indexConfigsByColName = FieldIndexConfigsUtil.createIndexConfigsByColName(tableConfig, schema); // Accessing the StarTree index configs is not handled by IndexType.getConfig(), so we manually update them. - if (tableConfig != null) { - IndexingConfig indexingConfig = tableConfig.getIndexingConfig(); - _enableDynamicStarTreeCreation = indexingConfig.isEnableDynamicStarTreeCreation(); - _starTreeIndexConfigs = indexingConfig.getStarTreeIndexConfigs(); - _enableDefaultStarTree = indexingConfig.isEnableDefaultStarTree(); - } + IndexingConfig indexingConfig = tableConfig.getIndexingConfig(); + _enableDynamicStarTreeCreation = indexingConfig.isEnableDynamicStarTreeCreation(); + _starTreeIndexConfigs = indexingConfig.getStarTreeIndexConfigs(); + _enableDefaultStarTree = indexingConfig.isEnableDefaultStarTree(); _dirty = false; } - /** - * Calculates the map from column to {@link FieldIndexConfigs}, merging the information related to older configs ( - * which is also heavily used by tests) and the one included in the TableConfig (in case the latter is not null). - * - * This method does not modify the result of {@link #getFieldIndexConfigByColName()} or - * {@link #getFieldIndexConfigByColName()}. To do so, call {@link #refreshIndexConfigs()}. - * - * The main difference between this method and - * {@link FieldIndexConfigsUtil#createIndexConfigsByColName(TableConfig, Schema)} is that the former relays - * on the TableConfig, while this method can be used even when the {@link IndexLoadingConfig} was configured by - * calling the setter methods. - */ - public Map calculateIndexConfigsByColName() { - return calculateIndexConfigsByColName(getTableConfigWithTierOverwrites(), inferSchema()); - } - - private Map calculateIndexConfigsByColName(@Nullable TableConfig tableConfig, - Schema schema) { - return FieldIndexConfigsUtil.createIndexConfigsByColName(tableConfig, schema, this::getDeserializer); - } - - private ColumnConfigDeserializer getDeserializer(IndexType indexType) { - ColumnConfigDeserializer deserializer; - - ColumnConfigDeserializer stdDeserializer = indexType::getConfig; - if (indexType instanceof ConfigurableFromIndexLoadingConfig) { - @SuppressWarnings("unchecked") - Map fromIndexLoadingConfig = - ((ConfigurableFromIndexLoadingConfig) indexType).fromIndexLoadingConfig(this); - - if (_schema == null || _tableConfig == null) { - LOGGER.debug("Ignoring default deserializers given that there is no schema [{}] or table config [{}]. Using " - + "indexLoadingConfig for indexType: {}", _schema == null, _tableConfig == null, indexType); - deserializer = IndexConfigDeserializer.fromMap(table -> fromIndexLoadingConfig); - } else if (_segmentTier == null) { - deserializer = - IndexConfigDeserializer.fromMap(table -> fromIndexLoadingConfig).withFallbackAlternative(stdDeserializer); - } else { - // No need to fall back to fromIndexLoadingConfig which contains index configs for default tier, when looking - // for tier specific index configs. - deserializer = stdDeserializer; - } - } else { - if (_schema == null || _tableConfig == null) { - LOGGER.debug( - "Ignoring default deserializers given that there is no schema [{}] or table config [{}]. Using default " - + "configs for indexType: {}", _schema == null, _tableConfig == null, indexType); - deserializer = (tableConfig, schema) -> getAllKnownColumns().stream() - .collect(Collectors.toMap(Function.identity(), col -> indexType.getDefaultConfig())); - } else { - deserializer = stdDeserializer; - } - } - return deserializer; - } - private TableConfig getTableConfigWithTierOverwrites() { return (_segmentTier == null || _tableConfig == null) ? _tableConfig : TableConfigUtils.overwriteTableConfigForTier(_tableConfig, _segmentTier); @@ -330,73 +329,10 @@ private Schema inferSchema() { return schema; } - /** - * Text index creation info for each column is specified - * using {@link FieldConfig} model of indicating per column - * encoding and indexing information. Since IndexLoadingConfig - * is created from TableConfig, we extract the text index info - * from fieldConfigList in TableConfig. - * @param tableConfig table config - */ - private void extractTextIndexColumnsFromTableConfig(TableConfig tableConfig) { - List fieldConfigList = tableConfig.getFieldConfigList(); - if (fieldConfigList != null) { - for (FieldConfig fieldConfig : fieldConfigList) { - if (fieldConfig.getIndexTypes().contains(FieldConfig.IndexType.TEXT)) { - _textIndexColumns.add(fieldConfig.getName()); - } - } - } - } - - private void extractFSTIndexColumnsFromTableConfig(TableConfig tableConfig) { - List fieldConfigList = tableConfig.getFieldConfigList(); - if (fieldConfigList != null) { - for (FieldConfig fieldConfig : fieldConfigList) { - if (fieldConfig.getIndexTypes().contains(FieldConfig.IndexType.FST)) { - _fstIndexColumns.add(fieldConfig.getName()); - } - } - } - } - - private void extractFromInstanceConfig(InstanceDataManagerConfig instanceDataManagerConfig) { - if (instanceDataManagerConfig == null) { - return; - } - - _instanceDataManagerConfig = instanceDataManagerConfig; - _instanceId = instanceDataManagerConfig.getInstanceId(); - - ReadMode instanceReadMode = instanceDataManagerConfig.getReadMode(); - if (instanceReadMode != null) { - _readMode = instanceReadMode; - } - - String instanceSegmentVersion = instanceDataManagerConfig.getSegmentFormatVersion(); - if (instanceSegmentVersion != null) { - _segmentVersion = SegmentVersion.valueOf(instanceSegmentVersion.toLowerCase()); - } - - _isRealtimeOffHeapAllocation = instanceDataManagerConfig.isRealtimeOffHeapAllocation(); - _isDirectRealtimeOffHeapAllocation = instanceDataManagerConfig.isDirectRealtimeOffHeapAllocation(); - - String avgMultiValueCount = instanceDataManagerConfig.getAvgMultiValueCount(); - if (avgMultiValueCount != null) { - _realtimeAvgMultiValueCount = Integer.parseInt(avgMultiValueCount); - } - _segmentStoreURI = - instanceDataManagerConfig.getConfig().getProperty(CommonConstants.Server.CONFIG_OF_SEGMENT_STORE_URI); - _segmentDirectoryLoader = instanceDataManagerConfig.getSegmentDirectoryLoader(); - } - public ReadMode getReadMode() { return _readMode; } - /** - * For tests only. - */ public void setReadMode(ReadMode readMode) { _readMode = readMode; _dirty = true; @@ -446,83 +382,6 @@ public Map> getColumnProperties() { return unmodifiable(_columnProperties); } - @Deprecated - @VisibleForTesting - public void setColumnProperties(Map> columnProperties) { - _columnProperties = new HashMap<>(columnProperties); - _dirty = true; - } - - @Deprecated - @VisibleForTesting - public void setInvertedIndexColumns(Set invertedIndexColumns) { - _invertedIndexColumns = new HashSet<>(invertedIndexColumns); - _dirty = true; - } - - @Deprecated - @VisibleForTesting - public void addNoDictionaryColumns(Collection noDictionaryColumns) { - _noDictionaryColumns.addAll(noDictionaryColumns); - _dirty = true; - } - - @Deprecated - @VisibleForTesting - public void setRangeIndexColumns(Set rangeIndexColumns) { - _rangeIndexColumns = new HashSet<>(rangeIndexColumns); - _dirty = true; - } - - @Deprecated - @VisibleForTesting - public void setTextIndexColumns(Set textIndexColumns) { - _textIndexColumns = new HashSet<>(textIndexColumns); - _dirty = true; - } - - @Deprecated - @VisibleForTesting - public void setFSTIndexColumns(Set fstIndexColumns) { - _fstIndexColumns = new HashSet<>(fstIndexColumns); - _dirty = true; - } - - @Deprecated - @VisibleForTesting - public void setFSTIndexType(FSTType fstType) { - _fstIndexType = fstType; - _dirty = true; - } - - @Deprecated - @VisibleForTesting - public void setJsonIndexColumns(Set jsonIndexColumns) { - if (jsonIndexColumns != null) { - _jsonIndexConfigs = new HashMap<>(); - for (String jsonIndexColumn : jsonIndexColumns) { - _jsonIndexConfigs.put(jsonIndexColumn, new JsonIndexConfig()); - } - } else { - _jsonIndexConfigs = null; - } - _dirty = true; - } - - @Deprecated - @VisibleForTesting - public void setBloomFilterConfigs(Map bloomFilterConfigs) { - _bloomFilterConfigs = new HashMap<>(bloomFilterConfigs); - _dirty = true; - } - - @Deprecated - @VisibleForTesting - public void setOnHeapDictionaryColumns(Set onHeapDictionaryColumns) { - _onHeapDictionaryColumns = new HashSet<>(onHeapDictionaryColumns); - _dirty = true; - } - public Set getNoDictionaryColumns() { return unmodifiable(_noDictionaryColumns); } @@ -598,16 +457,6 @@ public int getRealtimeAvgMultiValueCount() { return _realtimeAvgMultiValueCount; } - @Nullable - public TableConfig getTableConfig() { - return _tableConfig; - } - - @Nullable - public Schema getSchema() { - return _schema; - } - public String getSegmentDirectoryLoader() { return StringUtils.isNotBlank(_segmentDirectoryLoader) ? _segmentDirectoryLoader : SegmentDirectoryLoaderRegistry.DEFAULT_SEGMENT_DIRECTORY_LOADER_NAME; @@ -623,30 +472,29 @@ public String getInstanceId() { return _instanceId; } - public void setTableDataDir(String tableDataDir) { - _tableDataDir = tableDataDir; - _dirty = true; - } - - public boolean isErrorOnColumnBuildFailure() { - return _errorOnColumnBuildFailure; + public String getSegmentTier() { + return _segmentTier; } - public void setErrorOnColumnBuildFailure(boolean errorOnColumnBuildFailure) { - _errorOnColumnBuildFailure = errorOnColumnBuildFailure; + public void setSegmentTier(String segmentTier) { + _segmentTier = segmentTier; + _dirty = true; } public String getTableDataDir() { return _tableDataDir; } - public void setSegmentTier(String segmentTier) { - _segmentTier = segmentTier; - _dirty = true; + public void setTableDataDir(String tableDataDir) { + _tableDataDir = tableDataDir; } - public String getSegmentTier() { - return _segmentTier; + public boolean isErrorOnColumnBuildFailure() { + return _errorOnColumnBuildFailure; + } + + public void setErrorOnColumnBuildFailure(boolean errorOnColumnBuildFailure) { + _errorOnColumnBuildFailure = errorOnColumnBuildFailure; } @Nullable diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/map/MapIndexType.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/map/MapIndexType.java index c7e5bda1133..c8a2ac4b09c 100644 --- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/map/MapIndexType.java +++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/map/MapIndexType.java @@ -77,15 +77,14 @@ public String getPrettyName() { @Override public ColumnConfigDeserializer createDeserializer() { - // reads tableConfig.indexingConfig.mapIndexConfigs - ColumnConfigDeserializer fromMapIndexConf = + ColumnConfigDeserializer fromIndexes = + IndexConfigDeserializer.fromIndexes(getPrettyName(), getIndexConfigClass()); + ColumnConfigDeserializer fromMapIndexConfigs = IndexConfigDeserializer.fromMap(tableConfig -> tableConfig.getIndexingConfig().getMapIndexConfigs()); - // reads tableConfig.indexingConfig.mapIndexColumns - ColumnConfigDeserializer fromMapIndexCols = + ColumnConfigDeserializer fromMapIndexColumns = IndexConfigDeserializer.fromCollection(tableConfig -> tableConfig.getIndexingConfig().getMapIndexColumns(), - (accum, column) -> accum.put(column, new MapIndexConfig())); - return IndexConfigDeserializer.fromIndexes(getPrettyName(), getIndexConfigClass()).withExclusiveAlternative( - IndexConfigDeserializer.ifIndexingConfig(fromMapIndexCols.withExclusiveAlternative(fromMapIndexConf))); + (accum, column) -> accum.put(column, MapIndexConfig.DEFAULT)); + return fromIndexes.withExclusiveAlternative(fromMapIndexConfigs.withFallbackAlternative(fromMapIndexColumns)); } @Override diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/range/RangeIndexType.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/range/RangeIndexType.java index 6cf671eff37..43c6332f5c7 100644 --- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/range/RangeIndexType.java +++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/range/RangeIndexType.java @@ -24,13 +24,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; import javax.annotation.Nullable; import org.apache.pinot.segment.local.segment.creator.impl.inv.BitSlicedRangeIndexCreator; import org.apache.pinot.segment.local.segment.creator.impl.inv.RangeIndexCreator; -import org.apache.pinot.segment.local.segment.index.loader.ConfigurableFromIndexLoadingConfig; -import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig; import org.apache.pinot.segment.local.segment.index.loader.invertedindex.RangeIndexHandler; import org.apache.pinot.segment.local.segment.index.readers.BitSlicedRangeIndexReader; import org.apache.pinot.segment.local.segment.index.readers.RangeIndexReaderImpl; @@ -56,8 +52,8 @@ import org.apache.pinot.spi.data.Schema; -public class RangeIndexType extends AbstractIndexType - implements ConfigurableFromIndexLoadingConfig { +public class RangeIndexType + extends AbstractIndexType { public static final String INDEX_DISPLAY_NAME = "range"; private static final List EXTENSIONS = Collections.singletonList(V1Constants.Indexes.BITMAP_RANGE_INDEX_FILE_EXTENSION); @@ -71,15 +67,6 @@ public Class getIndexConfigClass() { return RangeIndexConfig.class; } - @Override - public Map fromIndexLoadingConfig(IndexLoadingConfig indexLoadingConfig) { - int rangeVersion = indexLoadingConfig.getRangeIndexVersion(); - return indexLoadingConfig.getRangeIndexColumns().stream() - .collect(Collectors.toMap( - Function.identity(), - c -> new RangeIndexConfig(rangeVersion))); - } - @Override public RangeIndexConfig getDefaultConfig() { return RangeIndexConfig.DISABLED; diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/text/TextIndexType.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/text/TextIndexType.java index 7a936aac764..3c2c8e6aa03 100644 --- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/text/TextIndexType.java +++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/text/TextIndexType.java @@ -27,15 +27,11 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; import javax.annotation.Nullable; import org.apache.pinot.segment.local.realtime.impl.invertedindex.NativeMutableTextIndex; import org.apache.pinot.segment.local.realtime.impl.invertedindex.RealtimeLuceneTextIndex; import org.apache.pinot.segment.local.segment.creator.impl.text.LuceneTextIndexCreator; import org.apache.pinot.segment.local.segment.creator.impl.text.NativeTextIndexCreator; -import org.apache.pinot.segment.local.segment.index.loader.ConfigurableFromIndexLoadingConfig; -import org.apache.pinot.segment.local.segment.index.loader.IndexLoadingConfig; import org.apache.pinot.segment.local.segment.index.loader.invertedindex.TextIndexHandler; import org.apache.pinot.segment.local.segment.index.readers.text.LuceneTextIndexReader; import org.apache.pinot.segment.local.segment.index.readers.text.NativeTextIndexReader; @@ -66,8 +62,7 @@ import org.slf4j.LoggerFactory; -public class TextIndexType extends AbstractIndexType - implements ConfigurableFromIndexLoadingConfig { +public class TextIndexType extends AbstractIndexType { protected static final Logger LOGGER = LoggerFactory.getLogger(TextIndexType.class); public static final String INDEX_DISPLAY_NAME = "text"; @@ -88,17 +83,6 @@ public Class getIndexConfigClass() { return TextIndexConfig.class; } - @Override - public Map fromIndexLoadingConfig(IndexLoadingConfig indexLoadingConfig) { - Map> allColProps = indexLoadingConfig.getColumnProperties(); - return indexLoadingConfig.getTextIndexColumns().stream().collect(Collectors.toMap( - Function.identity(), - colName -> new TextIndexConfigBuilder(indexLoadingConfig.getFSTIndexType()) - .withProperties(allColProps.get(colName)) - .build() - )); - } - @Override public TextIndexConfig getDefaultConfig() { return TextIndexConfig.DISABLED; @@ -165,11 +149,10 @@ private static class ReaderFactory implements IndexReaderFactory 0 ? _indexTypes.get(0) : null; + return !_indexTypes.isEmpty() ? _indexTypes.get(0) : null; } public List getIndexTypes() { @@ -206,7 +205,6 @@ public Map getProperties() { } public static class Builder { - @Nonnull private String _name; private EncodingType _encodingType; private List _indexTypes; @@ -216,7 +214,7 @@ public static class Builder { private TimestampConfig _timestampConfig; private JsonNode _tierOverwrites; - public Builder(@Nonnull String name) { + public Builder(String name) { _name = name; } diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/JsonIndexConfig.java b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/JsonIndexConfig.java index 1a0964138ed..4dbbf6c07d2 100644 --- a/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/JsonIndexConfig.java +++ b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/JsonIndexConfig.java @@ -45,6 +45,7 @@ * and continue indexing on following Json records. */ public class JsonIndexConfig extends IndexConfig { + public static final JsonIndexConfig DEFAULT = new JsonIndexConfig(); public static final JsonIndexConfig DISABLED = new JsonIndexConfig(true); private int _maxLevels = -1; diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/MapIndexConfig.java b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/MapIndexConfig.java index 58eff85ab95..9acb3032a84 100644 --- a/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/MapIndexConfig.java +++ b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/MapIndexConfig.java @@ -30,6 +30,7 @@ * Configs related to the MAP index: */ public class MapIndexConfig extends IndexConfig { + public static final MapIndexConfig DEFAULT = new MapIndexConfig(); public static final MapIndexConfig DISABLED = new MapIndexConfig(true); private final Map _configs;