Remove searchable_snapshot.extended_compatibility experimental feature

server/src/main/java/org/opensearch/common/util/FeatureFlags.java

@@ -36,18 +36,6 @@ public class FeatureFlags {
      */
     public static final String REMOTE_STORE_MIGRATION_EXPERIMENTAL = FEATURE_FLAG_PREFIX + "remote_store.migration.enabled";
 
-    /**
-     * Gates the ability for Searchable Snapshots to read snapshots that are older than the
-     * guaranteed backward compatibility for OpenSearch (one prior major version) on a best effort basis.
-     */
-    public static final String SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY = FEATURE_FLAG_PREFIX
-        + "searchable_snapshot.extended_compatibility.enabled";
-    public static final Setting<Boolean> SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY_SETTING = Setting.boolSetting(
-        SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY,
-        false,
-        Property.NodeScope
-    );
-
     /**
      * Gates the functionality of extensions.
      * Once the feature is ready for production release, this feature flag can be removed.
@@ -153,10 +141,6 @@ static class FeatureFlagsImpl {
                 );
                 put(TERM_VERSION_PRECOMMIT_ENABLE_SETTING, TERM_VERSION_PRECOMMIT_ENABLE_SETTING.getDefault(Settings.EMPTY));
                 put(ARROW_STREAMS_SETTING, ARROW_STREAMS_SETTING.getDefault(Settings.EMPTY));
-                put(
-                    SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY_SETTING,
-                    SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY_SETTING.getDefault(Settings.EMPTY)
-                );
                 put(MERGED_SEGMENT_WARMER_EXPERIMENTAL_SETTING, MERGED_SEGMENT_WARMER_EXPERIMENTAL_SETTING.getDefault(Settings.EMPTY));
             }
         };

server/src/main/java/org/opensearch/index/IndexSettings.java

@@ -43,7 +43,6 @@
 import org.opensearch.common.settings.Setting.Property;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
-import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.unit.ByteSizeUnit;
 import org.opensearch.core.common.unit.ByteSizeValue;
@@ -70,14 +69,12 @@
 import java.util.function.UnaryOperator;
 
 import static org.opensearch.Version.V_2_7_0;
-import static org.opensearch.common.util.FeatureFlags.SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY;
 import static org.opensearch.index.codec.fuzzy.FuzzySetParameters.DEFAULT_FALSE_POSITIVE_PROBABILITY;
 import static org.opensearch.index.mapper.MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING;
 import static org.opensearch.index.mapper.MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING;
 import static org.opensearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING;
 import static org.opensearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING;
 import static org.opensearch.index.mapper.MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING;
-import static org.opensearch.index.store.remote.directory.RemoteSnapshotDirectory.SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY_MINIMUM_VERSION;
 import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_DEFAULT_SLICE_COUNT_VALUE;
 import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MIN_SLICE_COUNT_VALUE;
 import static org.opensearch.search.SearchService.CONCURRENT_SEGMENT_SEARCH_MODE_ALL;
@@ -805,7 +802,6 @@ public static IndexMergePolicy fromString(String text) {
     private volatile String remoteStoreRepository;
     private int remoteTranslogKeepExtraGen;
     private boolean autoForcemergeEnabled;
-    private Version extendedCompatibilitySnapshotVersion;
 
     // volatile fields are updated via #updateIndexMetadata(IndexMetadata) under lock
     private volatile Settings settings;
@@ -1019,13 +1015,6 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
         remoteTranslogUploadBufferInterval = INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(settings);
         remoteStoreRepository = settings.get(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY);
         this.remoteTranslogKeepExtraGen = INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING.get(settings);
-
-        if (isRemoteSnapshot() && FeatureFlags.isEnabled(SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY)) {
-            extendedCompatibilitySnapshotVersion = SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY_MINIMUM_VERSION;
-        } else {
-            extendedCompatibilitySnapshotVersion = Version.CURRENT.minimumIndexCompatibilityVersion();
-        }
-
         this.searchThrottled = INDEX_SEARCH_THROTTLED.get(settings);
         this.shouldCleanupUnreferencedFiles = INDEX_UNREFERENCED_FILE_CLEANUP.get(settings);
         this.queryStringLenient = QUERY_STRING_LENIENT_SETTING.get(settings);
@@ -1438,16 +1427,6 @@ public boolean isRemoteSnapshot() {
         return indexMetadata.isRemoteSnapshot();
     }
 
-    /**
-     * If this is a remote snapshot and the extended compatibility
-     * feature flag is enabled, this returns the minimum {@link Version}
-     * supported. In all other cases, the return value is the
-     * {@link Version#minimumIndexCompatibilityVersion()} of {@link Version#CURRENT}.
-     */
-    public Version getExtendedCompatibilitySnapshotVersion() {
-        return extendedCompatibilitySnapshotVersion;
-    }
-
     /**
      * Returns the node settings. The settings returned from {@link #getSettings()} are a merged version of the
      * index settings and the node settings where node settings are overwritten by index settings.

server/src/main/java/org/opensearch/index/engine/ReadOnlyEngine.java

@@ -39,7 +39,6 @@
 import org.apache.lucene.search.ReferenceManager;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.Lock;
-import org.opensearch.Version;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.concurrent.GatedCloseable;
 import org.opensearch.common.lucene.Lucene;
@@ -93,7 +92,6 @@ public class ReadOnlyEngine extends Engine {
     private final CompletionStatsCache completionStatsCache;
     private final boolean requireCompleteHistory;
     private final TranslogManager translogManager;
-    private final Version minimumSupportedVersion;
 
     protected volatile TranslogStats translogStats;
 
@@ -120,8 +118,6 @@ public ReadOnlyEngine(
     ) {
         super(config);
         this.requireCompleteHistory = requireCompleteHistory;
-        // fetch the minimum Version for extended backward compatibility use-cases
-        this.minimumSupportedVersion = config.getIndexSettings().getExtendedCompatibilitySnapshotVersion();
         try {
             Store store = config.getStore();
             store.incRef();
@@ -133,11 +129,7 @@ public ReadOnlyEngine(
                 // we obtain the IW lock even though we never modify the index.
                 // yet this makes sure nobody else does. including some testing tools that try to be messy
                 indexWriterLock = obtainLock ? directory.obtainLock(IndexWriter.WRITE_LOCK_NAME) : null;
-                if (isExtendedCompatibility()) {
-                    this.lastCommittedSegmentInfos = Lucene.readSegmentInfos(directory, this.minimumSupportedVersion);
-                } else {
-                    this.lastCommittedSegmentInfos = Lucene.readSegmentInfos(directory);
-                }
+                this.lastCommittedSegmentInfos = Lucene.readSegmentInfos(directory);
                 if (seqNoStats == null) {
                     seqNoStats = buildSeqNoStats(config, lastCommittedSegmentInfos);
                     ensureMaxSeqNoEqualsToGlobalCheckpoint(seqNoStats);
@@ -221,19 +213,10 @@ protected final OpenSearchDirectoryReader wrapReader(
 
     protected DirectoryReader open(IndexCommit commit) throws IOException {
         assert Transports.assertNotTransportThread("opening index commit of a read-only engine");
-        DirectoryReader reader;
-        if (isExtendedCompatibility()) {
-            reader = DirectoryReader.open(commit, this.minimumSupportedVersion.luceneVersion.major, null);
-        } else {
-            reader = DirectoryReader.open(commit);
-        }
+        DirectoryReader reader = DirectoryReader.open(commit);
         return new SoftDeletesDirectoryReaderWrapper(reader, Lucene.SOFT_DELETES_FIELD);
     }
 
-    private boolean isExtendedCompatibility() {
-        return Version.CURRENT.minimumIndexCompatibilityVersion().onOrAfter(this.minimumSupportedVersion);
-    }
-
     @Override
     protected void closeNoLock(String reason, CountDownLatch closedLatch) {
         if (isClosed.compareAndSet(false, true)) {

server/src/main/java/org/opensearch/index/shard/IndexShard.java

@@ -2772,11 +2772,7 @@ private boolean assertSequenceNumbersInCommit() throws IOException {
     }
 
     private Map<String, String> fetchUserData() throws IOException {
-        if (indexSettings.isRemoteSnapshot() && indexSettings.getExtendedCompatibilitySnapshotVersion() != null) {
-            return Lucene.readSegmentInfos(store.directory(), indexSettings.getExtendedCompatibilitySnapshotVersion()).getUserData();
-        } else {
-            return SegmentInfos.readLatestCommit(store.directory()).getUserData();
-        }
+        return SegmentInfos.readLatestCommit(store.directory()).getUserData();
     }
 
     private void onNewEngine(Engine newEngine) {

server/src/main/java/org/opensearch/index/store/Store.java

@@ -231,11 +231,7 @@ public ShardPath shardPath() {
     public SegmentInfos readLastCommittedSegmentsInfo() throws IOException {
         failIfCorrupted();
         try {
-            if (indexSettings.isRemoteSnapshot() && indexSettings.getExtendedCompatibilitySnapshotVersion() != null) {
-                return readSegmentInfosExtendedCompatibility(directory(), indexSettings.getExtendedCompatibilitySnapshotVersion());
-            } else {
-                return readSegmentsInfo(null, directory());
-            }
+            return readSegmentsInfo(null, directory());
         } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
             markStoreCorrupted(ex);
             throw ex;
@@ -245,7 +241,7 @@ public SegmentInfos readLastCommittedSegmentsInfo() throws IOException {
     /**
      * Returns the segments info for the given commit or for the latest commit if the given commit is <code>null</code>.
      * This method will throw an exception if the index is older than the standard backwards compatibility
-     * policy ( current major - 1). See also {@link #readSegmentInfosExtendedCompatibility(Directory, org.opensearch.Version)}.
+     * policy ( current major - 1).
      *
      * @throws IOException if the index is corrupted or the segments file is not present
      */
@@ -263,27 +259,6 @@ private static SegmentInfos readSegmentsInfo(IndexCommit commit, Directory direc
         }
     }
 
-    /**
-     * Returns the segments info for the latest commit in the given directory. Unlike
-     * {@link #readSegmentsInfo(IndexCommit, Directory)}, this method supports reading
-     * older Lucene indices on a best-effort basis.
-     *
-     * @throws IOException if the index is corrupted or the segments file is not present
-     */
-    private static SegmentInfos readSegmentInfosExtendedCompatibility(Directory directory, org.opensearch.Version minimumVersion)
-        throws IOException {
-        try {
-            return Lucene.readSegmentInfos(directory, minimumVersion);
-        } catch (EOFException eof) {
-            // TODO this should be caught by lucene - EOF is almost certainly an index corruption
-            throw new CorruptIndexException("Read past EOF while reading segment infos", "<latest-commit>", eof);
-        } catch (IOException exception) {
-            throw exception; // IOExceptions like too many open files are not necessarily a corruption - just bubble it up
-        } catch (Exception ex) {
-            throw new CorruptIndexException("Hit unexpected exception while reading segment infos", "<latest-commit>", ex);
-        }
-    }
-
     final void ensureOpen() {
         if (this.refCounter.refCount() <= 0) {
             throw new AlreadyClosedException("store is already closed");

server/src/main/java/org/opensearch/index/store/remote/directory/RemoteSnapshotDirectory.java

@@ -15,8 +15,6 @@
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.store.Lock;
 import org.apache.lucene.store.NoLockFactory;
-import org.opensearch.LegacyESVersion;
-import org.opensearch.Version;
 import org.opensearch.common.lucene.store.ByteArrayIndexInput;
 import org.opensearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot;
 import org.opensearch.index.store.remote.file.OnDemandBlockSnapshotIndexInput;
@@ -38,8 +36,6 @@
  */
 public final class RemoteSnapshotDirectory extends Directory {
 
-    public static final Version SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY_MINIMUM_VERSION = LegacyESVersion.V_7_2_0;
-
     private static final String VIRTUAL_FILE_PREFIX = BlobStoreRepository.VIRTUAL_DATA_BLOB_PREFIX;
 
     private final Map<String, BlobStoreIndexShardSnapshot.FileInfo> fileInfoMap;

server/src/main/java/org/opensearch/snapshots/RestoreService.java

@@ -80,7 +80,6 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.util.ArrayUtils;
-import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.index.Index;
 import org.opensearch.core.index.shard.ShardId;
@@ -126,11 +125,9 @@
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_VERSION_UPGRADED;
 import static org.opensearch.cluster.service.ClusterManagerTask.RESTORE_SNAPSHOT;
-import static org.opensearch.common.util.FeatureFlags.SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY;
 import static org.opensearch.common.util.IndexUtils.filterIndices;
 import static org.opensearch.common.util.set.Sets.newHashSet;
 import static org.opensearch.index.IndexModule.INDEX_STORE_TYPE_SETTING;
-import static org.opensearch.index.store.remote.directory.RemoteSnapshotDirectory.SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY_MINIMUM_VERSION;
 import static org.opensearch.node.Node.NODE_SEARCH_CACHE_SIZE_SETTING;
 
 /**
@@ -434,15 +431,9 @@ public ClusterState execute(ClusterState currentState) {
                                     request.getSourceRemoteTranslogRepository(),
                                     snapshotInfo.getPinnedTimestamp()
                                 );
-                                final Version minIndexCompatibilityVersion;
-                                if (isSearchableSnapshot && isSearchableSnapshotsExtendedCompatibilityEnabled()) {
-                                    minIndexCompatibilityVersion = SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY_MINIMUM_VERSION
-                                        .minimumIndexCompatibilityVersion();
-                                } else {
-                                    minIndexCompatibilityVersion = currentState.getNodes()
-                                        .getMaxNodeVersion()
-                                        .minimumIndexCompatibilityVersion();
-                                }
+                                final Version minIndexCompatibilityVersion = currentState.getNodes()
+                                    .getMaxNodeVersion()
+                                    .minimumIndexCompatibilityVersion();
                                 try {
                                     snapshotIndexMetadata = metadataIndexUpgradeService.upgradeIndexMetadata(
                                         snapshotIndexMetadata,
@@ -1390,9 +1381,4 @@ private static IndexMetadata addSnapshotToIndexSettings(IndexMetadata metadata,
             .build();
         return IndexMetadata.builder(metadata).settings(newSettings).build();
     }
-
-    private static boolean isSearchableSnapshotsExtendedCompatibilityEnabled() {
-        return org.opensearch.Version.CURRENT.after(org.opensearch.Version.V_2_4_0)
-            && FeatureFlags.isEnabled(SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY);
-    }
 }

server/src/test/java/org/opensearch/common/lucene/LuceneTests.java

@@ -36,13 +36,10 @@
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Field.Store;
 import org.apache.lucene.document.LatLonDocValuesField;
-import org.apache.lucene.document.LatLonPoint;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexCommit;
-import org.apache.lucene.index.IndexFormatTooOldException;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
@@ -51,7 +48,6 @@
 import org.apache.lucene.index.NoMergePolicy;
 import org.apache.lucene.index.SegmentInfos;
 import org.apache.lucene.index.SoftDeletesRetentionMergePolicy;
-import org.apache.lucene.index.StandardDirectoryReader;
 import org.apache.lucene.index.StoredFields;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.Explanation;
@@ -75,11 +71,8 @@
 import org.apache.lucene.tests.analysis.MockAnalyzer;
 import org.apache.lucene.tests.index.RandomIndexWriter;
 import org.apache.lucene.tests.store.MockDirectoryWrapper;
-import org.apache.lucene.tests.util.TestUtil;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.BytesRef;
-import org.opensearch.LegacyESVersion;
-import org.opensearch.Version;
 import org.opensearch.common.collect.Tuple;
 import org.opensearch.common.util.io.IOUtils;
 import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
@@ -95,7 +88,6 @@
 
 import java.io.IOException;
 import java.io.StringReader;
-import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
@@ -329,36 +321,6 @@ public void testNumDocs() throws IOException {
         dir.close();
     }
 
-    /**
-     * Tests whether old segments are readable and queryable based on the data documented
-     * in the README <a href="file:../../../../../resources/indices/bwc/os-1.3.0/README.md">here</a>.
-     */
-    public void testReadSegmentInfosExtendedCompatibility() throws IOException {
-        final Version minVersion = LegacyESVersion.V_7_2_0;
-        Path tmp = createTempDir();
-        TestUtil.unzip(getClass().getResourceAsStream(OLDER_VERSION_INDEX_ZIP_RELATIVE_PATH), tmp);
-        try (MockDirectoryWrapper dir = newMockFSDirectory(tmp)) {
-            // The standard API will throw an exception
-            expectThrows(IndexFormatTooOldException.class, () -> Lucene.readSegmentInfos(dir));
-            SegmentInfos si = Lucene.readSegmentInfos(dir, minVersion);
-            assertEquals(1, Lucene.getNumDocs(si));
-            IndexCommit indexCommit = Lucene.getIndexCommit(si, dir);
-            // uses the "expert" Lucene API
-            try (
-                StandardDirectoryReader reader = (StandardDirectoryReader) DirectoryReader.open(
-                    indexCommit,
-                    minVersion.minimumIndexCompatibilityVersion().luceneVersion.major,
-                    null
-                )
-            ) {
-                IndexSearcher searcher = newSearcher(reader);
-                // radius too small, should get no results
-                assertFalse(Lucene.exists(searcher, LatLonPoint.newDistanceQuery("testLocation", 48.57532, -112.87695, 2)));
-                assertTrue(Lucene.exists(searcher, LatLonPoint.newDistanceQuery("testLocation", 48.57532, -112.87695, 20000)));
-            }
-        }
-    }
-
     public void testCount() throws Exception {
         Directory dir = newDirectory();
         RandomIndexWriter w = new RandomIndexWriter(random(), dir);