-
Notifications
You must be signed in to change notification settings - Fork 15
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Backport] [2.x] Update codecs to Apache Lucene 9.12.0 (#199)
* Update codecs to Apache Lucene 9.12.0 Signed-off-by: Andriy Redko <andriy.redko@aiven.io> * Address code review comments Signed-off-by: Andriy Redko <andriy.redko@aiven.io> * Address code review comments Signed-off-by: Andriy Redko <andriy.redko@aiven.io> * Fix javadoc comments Signed-off-by: Andriy Redko <andriy.redko@aiven.io> --------- Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
- Loading branch information
Showing
33 changed files
with
1,150 additions
and
202 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
114 changes: 114 additions & 0 deletions
114
src/main/java/org/opensearch/index/codec/customcodecs/Lucene912CustomCodec.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,114 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.index.codec.customcodecs; | ||
|
|
||
| import org.apache.logging.log4j.Logger; | ||
| import org.apache.lucene.codecs.FilterCodec; | ||
| import org.apache.lucene.codecs.StoredFieldsFormat; | ||
| import org.apache.lucene.codecs.lucene912.Lucene912Codec; | ||
| import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec; | ||
| import org.opensearch.index.mapper.MapperService; | ||
|
|
||
| import java.util.Set; | ||
|
|
||
| import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; | ||
|
|
||
| /** | ||
| * | ||
| * Extends {@link FilterCodec} to reuse the functionality of Lucene Codec. | ||
| * Supports two modes zstd and zstd_no_dict. | ||
| * Uses Lucene912 as the delegate codec | ||
| * | ||
| * @opensearch.internal | ||
| */ | ||
| public abstract class Lucene912CustomCodec extends FilterCodec { | ||
|
|
||
| /** Each mode represents a compression algorithm. */ | ||
| public enum Mode { | ||
| /** | ||
| * ZStandard mode with dictionary | ||
| */ | ||
| ZSTD("ZSTD912", Set.of("zstd")), | ||
| /** | ||
| * ZStandard mode without dictionary | ||
| */ | ||
| ZSTD_NO_DICT("ZSTDNODICT912", Set.of("zstd_no_dict")); | ||
|
|
||
| private final String codec; | ||
| private final Set<String> aliases; | ||
|
|
||
| Mode(String codec, Set<String> aliases) { | ||
| this.codec = codec; | ||
| this.aliases = aliases; | ||
| } | ||
|
|
||
| /** | ||
| * Returns the Codec that is registered with Lucene | ||
| */ | ||
| public String getCodec() { | ||
| return codec; | ||
| } | ||
|
|
||
| /** | ||
| * Returns the aliases of the Codec | ||
| */ | ||
| public Set<String> getAliases() { | ||
| return aliases; | ||
| } | ||
| } | ||
|
|
||
| private final StoredFieldsFormat storedFieldsFormat; | ||
|
|
||
| /** | ||
| * Creates a new compression codec with the default compression level. | ||
| * | ||
| * @param mode The compression codec (ZSTD or ZSTDNODICT). | ||
| */ | ||
| public Lucene912CustomCodec(Mode mode) { | ||
| this(mode, DEFAULT_COMPRESSION_LEVEL); | ||
| } | ||
|
|
||
| /** | ||
| * Creates a new compression codec with the given compression level. We use | ||
| * lowercase letters when registering the codec so that we remain consistent with | ||
| * the other compression codecs: default, lucene_default, and best_compression. | ||
| * | ||
| * @param mode The compression codec (ZSTD or ZSTDNODICT). | ||
| * @param compressionLevel The compression level. | ||
| */ | ||
| public Lucene912CustomCodec(Mode mode, int compressionLevel) { | ||
| super(mode.getCodec(), new Lucene912Codec()); | ||
| this.storedFieldsFormat = new Lucene912CustomStoredFieldsFormat(mode, compressionLevel); | ||
| } | ||
|
|
||
| /** | ||
| * Creates a new compression codec with the given compression level. We use | ||
| * lowercase letters when registering the codec so that we remain consistent with | ||
| * the other compression codecs: default, lucene_default, and best_compression. | ||
| * | ||
| * @param mode The compression codec (ZSTD or ZSTDNODICT). | ||
| * @param compressionLevel The compression level. | ||
| * @param mapperService The mapper service. | ||
| * @param logger The logger. | ||
| */ | ||
| public Lucene912CustomCodec(Mode mode, int compressionLevel, MapperService mapperService, Logger logger) { | ||
| super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene912Codec.Mode.BEST_SPEED, mapperService, logger)); | ||
| this.storedFieldsFormat = new Lucene912CustomStoredFieldsFormat(mode, compressionLevel); | ||
| } | ||
|
|
||
| @Override | ||
| public StoredFieldsFormat storedFieldsFormat() { | ||
| return storedFieldsFormat; | ||
| } | ||
|
|
||
| @Override | ||
| public String toString() { | ||
| return getClass().getSimpleName(); | ||
| } | ||
| } |
140 changes: 140 additions & 0 deletions
140
src/main/java/org/opensearch/index/codec/customcodecs/Lucene912CustomStoredFieldsFormat.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,140 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.index.codec.customcodecs; | ||
|
|
||
| import org.apache.lucene.codecs.StoredFieldsFormat; | ||
| import org.apache.lucene.codecs.StoredFieldsReader; | ||
| import org.apache.lucene.codecs.StoredFieldsWriter; | ||
| import org.apache.lucene.codecs.compressing.CompressionMode; | ||
| import org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsFormat; | ||
| import org.apache.lucene.index.FieldInfos; | ||
| import org.apache.lucene.index.SegmentInfo; | ||
| import org.apache.lucene.store.Directory; | ||
| import org.apache.lucene.store.IOContext; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.Objects; | ||
|
|
||
| import static org.opensearch.index.codec.customcodecs.backward_codecs.lucene99.Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; | ||
|
|
||
| /** Stored field format used by pluggable codec */ | ||
| public class Lucene912CustomStoredFieldsFormat extends StoredFieldsFormat { | ||
|
|
||
| /** A key that we use to map to a mode */ | ||
| public static final String MODE_KEY = Lucene912CustomStoredFieldsFormat.class.getSimpleName() + ".mode"; | ||
|
|
||
| protected static final int ZSTD_BLOCK_LENGTH = 10 * 48 * 1024; | ||
| protected static final int ZSTD_MAX_DOCS_PER_BLOCK = 4096; | ||
| protected static final int ZSTD_BLOCK_SHIFT = 10; | ||
|
|
||
| private final CompressionMode zstdCompressionMode; | ||
| private final CompressionMode zstdNoDictCompressionMode; | ||
|
|
||
| private final Lucene912CustomCodec.Mode mode; | ||
| private final int compressionLevel; | ||
|
|
||
| /** default constructor */ | ||
| public Lucene912CustomStoredFieldsFormat() { | ||
| this(Lucene912CustomCodec.Mode.ZSTD, DEFAULT_COMPRESSION_LEVEL); | ||
| } | ||
|
|
||
| /** | ||
| * Creates a new instance. | ||
| * | ||
| * @param mode The mode represents ZSTD or ZSTDNODICT | ||
| */ | ||
| public Lucene912CustomStoredFieldsFormat(Lucene912CustomCodec.Mode mode) { | ||
| this(mode, DEFAULT_COMPRESSION_LEVEL); | ||
| } | ||
|
|
||
| /** | ||
| * Creates a new instance with the specified mode and compression level. | ||
| * | ||
| * @param mode The mode represents ZSTD or ZSTDNODICT | ||
| * @param compressionLevel The compression level for the mode. | ||
| */ | ||
| public Lucene912CustomStoredFieldsFormat(Lucene912CustomCodec.Mode mode, int compressionLevel) { | ||
| this.mode = Objects.requireNonNull(mode); | ||
| this.compressionLevel = compressionLevel; | ||
| zstdCompressionMode = new ZstdCompressionMode(compressionLevel); | ||
| zstdNoDictCompressionMode = new ZstdNoDictCompressionMode(compressionLevel); | ||
| } | ||
|
|
||
| /** | ||
| * Returns a {@link StoredFieldsReader} to load stored fields. | ||
| * @param directory The index directory. | ||
| * @param si The SegmentInfo that stores segment information. | ||
| * @param fn The fieldInfos. | ||
| * @param context The IOContext that holds additional details on the merge/search context. | ||
| */ | ||
| @Override | ||
| public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException { | ||
| if (si.getAttribute(MODE_KEY) != null) { | ||
| String value = si.getAttribute(MODE_KEY); | ||
| Lucene912CustomCodec.Mode mode = Lucene912CustomCodec.Mode.valueOf(value); | ||
| return impl(mode).fieldsReader(directory, si, fn, context); | ||
| } else { | ||
| throw new IllegalStateException("missing value for " + MODE_KEY + " for segment: " + si.name); | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Returns a {@link StoredFieldsReader} to write stored fields. | ||
| * @param directory The index directory. | ||
| * @param si The SegmentInfo that stores segment information. | ||
| * @param context The IOContext that holds additional details on the merge/search context. | ||
| */ | ||
| @Override | ||
| public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context) throws IOException { | ||
| String previous = si.putAttribute(MODE_KEY, mode.name()); | ||
| if (previous != null && previous.equals(mode.name()) == false) { | ||
| throw new IllegalStateException( | ||
| "found existing value for " + MODE_KEY + " for segment: " + si.name + " old = " + previous + ", new = " + mode.name() | ||
| ); | ||
| } | ||
| return impl(mode).fieldsWriter(directory, si, context); | ||
| } | ||
|
|
||
| StoredFieldsFormat impl(Lucene912CustomCodec.Mode mode) { | ||
| switch (mode) { | ||
| case ZSTD: | ||
| return getCustomCompressingStoredFieldsFormat("CustomStoredFieldsZstd", this.zstdCompressionMode); | ||
| case ZSTD_NO_DICT: | ||
| return getCustomCompressingStoredFieldsFormat("CustomStoredFieldsZstdNoDict", this.zstdNoDictCompressionMode); | ||
| default: | ||
| throw new IllegalStateException("Unsupported compression mode: " + mode); | ||
| } | ||
| } | ||
|
|
||
| private StoredFieldsFormat getCustomCompressingStoredFieldsFormat(String formatName, CompressionMode compressionMode) { | ||
| return new Lucene90CompressingStoredFieldsFormat( | ||
| formatName, | ||
| compressionMode, | ||
| ZSTD_BLOCK_LENGTH, | ||
| ZSTD_MAX_DOCS_PER_BLOCK, | ||
| ZSTD_BLOCK_SHIFT | ||
| ); | ||
| } | ||
|
|
||
| public Lucene912CustomCodec.Mode getMode() { | ||
| return mode; | ||
| } | ||
|
|
||
| /** | ||
| * Returns the compression level. | ||
| */ | ||
| public int getCompressionLevel() { | ||
| return compressionLevel; | ||
| } | ||
|
|
||
| public CompressionMode getCompressionMode() { | ||
| return mode == Lucene912CustomCodec.Mode.ZSTD_NO_DICT ? zstdNoDictCompressionMode : zstdCompressionMode; | ||
| } | ||
|
|
||
| } |
Oops, something went wrong.