Use JNA to Speed up Snapshot Cache File Creation (#68687)

Use JNA to speed up snapshot cache file creation. Do this in `:server`
to bypass the security filter and move necessary bits of code to `:server`
to enable the logic.
Fall back to trying to create the file by writing zeros if anything except for
the step determining free disk space fails.

Author: original-brownbear

server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java

@@ -40,6 +40,7 @@
 import org.elasticsearch.node.InternalSettingsPreparer;
 import org.elasticsearch.node.Node;
 import org.elasticsearch.node.NodeValidationException;
+import org.elasticsearch.snapshots.SnapshotsService;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -168,6 +169,18 @@ private void setup(boolean addShutdownHook, Environment environment) throws Boot
                 BootstrapSettings.SYSTEM_CALL_FILTER_SETTING.get(settings),
                 BootstrapSettings.CTRLHANDLER_SETTING.get(settings));
 
+        final long cacheSize = SnapshotsService.SNAPSHOT_CACHE_SIZE_SETTING.get(settings).getBytes();
+        final long regionSize = SnapshotsService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.get(settings).getBytes();
+        final int numRegions = Math.toIntExact(cacheSize / regionSize);
+        final long fileSize = numRegions * regionSize;
+        if (fileSize > 0) {
+            try {
+                Natives.tryCreateCacheFile(environment, fileSize);
+            } catch (Exception e) {
+                throw new BootstrapException(e);
+            }
+        }
+
         // initialize probes before the security manager is installed
         initializeProbes();
 

server/src/main/java/org/elasticsearch/bootstrap/JNACLibrary.java

@@ -61,6 +61,9 @@ protected List<String> getFieldOrder() {
 
     static native String strerror(int errno);
 
+    // TODO: Bind POSIX fallocate as well to support non-Linux? (this would only apply to OSX in practice?)
+    static native int fallocate(int fd, int mode, long offset, long length);
+
     private JNACLibrary() {
     }
 }

server/src/main/java/org/elasticsearch/bootstrap/JNANatives.java

@@ -14,9 +14,17 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.lucene.util.Constants;
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.env.Environment;
 import org.elasticsearch.monitor.jvm.JvmInfo;
+import org.elasticsearch.snapshots.SnapshotUtils;
 
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.nio.file.Files;
 import java.nio.file.Path;
 
 import static org.elasticsearch.bootstrap.JNAKernel32Library.SizeT;
@@ -260,4 +268,39 @@ static void tryInstallSystemCallFilter(Path tmpFile) {
             logger.warn("unable to install syscall filter: ", e);
         }
     }
+
+    @SuppressForbidden(reason = "need access to fd on FileOutputStream")
+    static void fallocateSnapshotCacheFile(Environment environment, long fileSize) throws IOException {
+        if (Constants.LINUX == false) {
+            logger.debug("not trying to create a shared cache file using fallocate on non-Linux platform");
+            return;
+        }
+        Path cacheFile = SnapshotUtils.findCacheSnapshotCacheFilePath(environment, fileSize);
+        if (cacheFile == null) {
+            throw new IOException("could not find a directory with adequate free space for cache file");
+        }
+        boolean success = false;
+        try (FileOutputStream fileChannel = new FileOutputStream(cacheFile.toFile())) {
+            long currentSize = fileChannel.getChannel().size();
+            if (currentSize < fileSize) {
+                final Field field = fileChannel.getFD().getClass().getDeclaredField("fd");
+                field.setAccessible(true);
+                final int result = JNACLibrary.fallocate((int) field.get(fileChannel.getFD()), 0, currentSize, fileSize - currentSize);
+                final int errno = result == 0 ? 0 : Native.getLastError();
+                if (errno == 0) {
+                    success = true;
+                    logger.info("allocated cache file [{}] using fallocate", cacheFile);
+                } else {
+                    logger.warn("failed to initialize cache file [{}] using fallocate errno [{}]", cacheFile, errno);
+                }
+            }
+        } catch (Exception e) {
+            logger.warn(new ParameterizedMessage("failed to initialize cache file [{}] using fallocate", cacheFile), e);
+        } finally {
+            if (success == false) {
+                // if anything goes wrong, delete the potentially created file to not waste disk space
+                Files.deleteIfExists(cacheFile);
+            }
+        }
+    }
 }

server/src/main/java/org/elasticsearch/bootstrap/Natives.java

@@ -10,7 +10,9 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.elasticsearch.env.Environment;
 
+import java.io.IOException;
 import java.nio.file.Path;
 
 /**
@@ -132,4 +134,20 @@ static boolean isSystemCallFilterInstalled() {
         }
         return JNANatives.LOCAL_SYSTEM_CALL_FILTER;
     }
+
+    /**
+     * On Linux, this method tries to create the searchable snapshot frozen cache file using fallocate if JNA is available. This enables
+     * a much faster creation of the file than the fallback mechanism in the searchable snapshots plugin that will pre-allocate the cache
+     * file by writing zeros to the file.
+     *
+     * @throws IOException on failure to determine free disk space for a data path
+     */
+    public static void tryCreateCacheFile(Environment environment, long fileSize) throws IOException {
+        if (JNA_AVAILABLE == false) {
+            logger.warn("cannot use fallocate to create cache file because JNA is not available");
+            return;
+        }
+        JNANatives.fallocateSnapshotCacheFile(environment, fileSize);
+    }
+
 }

server/src/main/java/org/elasticsearch/env/Environment.java

@@ -300,6 +300,16 @@ public static FileStore getFileStore(final Path path) throws IOException {
         return new ESFileStore(Files.getFileStore(path));
     }
 
+    public static long getUsableSpace(Path path) throws IOException {
+        long freeSpaceInBytes = Environment.getFileStore(path).getUsableSpace();
+
+        /* See: https://bugs.openjdk.java.net/browse/JDK-8162520 */
+        if (freeSpaceInBytes < 0) {
+            freeSpaceInBytes = Long.MAX_VALUE;
+        }
+        return freeSpaceInBytes;
+    }
+
     /**
      * asserts that the two environments are equivalent for all things the environment cares about (i.e., all but the setting
      * object which may contain different setting)

server/src/main/java/org/elasticsearch/snapshots/SnapshotUtils.java

@@ -9,9 +9,14 @@
 
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.env.Environment;
 import org.elasticsearch.index.IndexNotFoundException;
 
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
@@ -107,4 +112,29 @@ public static List<String> filterIndices(List<String> availableIndices, String[]
         }
         return List.copyOf(result);
     }
+
+    /**
+     * Tries to find a suitable path to a searchable snapshots shared cache file in the data paths founds in the environment.
+     *
+     * @return path for the cache file or {@code null} if none could be found
+     */
+    @Nullable
+    public static Path findCacheSnapshotCacheFilePath(Environment environment, long fileSize) throws IOException {
+        Path cacheFile = null;
+        for (Path path : environment.dataFiles()) {
+            Files.createDirectories(path);
+            // TODO: be resilient to this check failing and try next path?
+            long usableSpace = Environment.getUsableSpace(path);
+            Path p = path.resolve(SnapshotsService.CACHE_FILE_NAME);
+            if (Files.exists(p)) {
+                usableSpace += Files.size(p);
+            }
+            // TODO: leave some margin for error here
+            if (usableSpace > fileSize) {
+                cacheFile = p;
+                break;
+            }
+        }
+        return cacheFile;
+    }
 }

server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java

@@ -37,6 +37,7 @@
 import org.elasticsearch.cluster.RestoreInProgress;
 import org.elasticsearch.cluster.SnapshotDeletionsInProgress;
 import org.elasticsearch.cluster.SnapshotsInProgress;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.repositories.RepositoryShardId;
 import org.elasticsearch.cluster.SnapshotsInProgress.ShardSnapshotStatus;
@@ -132,6 +133,26 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
 
     public static final String UPDATE_SNAPSHOT_STATUS_ACTION_NAME = "internal:cluster/snapshot/update_snapshot_status";
 
+    public static final String SHARED_CACHE_SETTINGS_PREFIX = "xpack.searchable.snapshot.shared_cache.";
+
+    public static final Setting<ByteSizeValue> SHARED_CACHE_RANGE_SIZE_SETTING = Setting.byteSizeSetting(
+        SHARED_CACHE_SETTINGS_PREFIX + "range_size",
+        ByteSizeValue.ofMb(16),                                 // default
+        Setting.Property.NodeScope
+    );
+    public static final Setting<ByteSizeValue> SNAPSHOT_CACHE_REGION_SIZE_SETTING = Setting.byteSizeSetting(
+        SHARED_CACHE_SETTINGS_PREFIX + "region_size",
+        SHARED_CACHE_RANGE_SIZE_SETTING,
+        Setting.Property.NodeScope
+    );
+    public static final Setting<ByteSizeValue> SNAPSHOT_CACHE_SIZE_SETTING = Setting.byteSizeSetting(
+        SHARED_CACHE_SETTINGS_PREFIX + "size",
+        ByteSizeValue.ZERO,
+        Setting.Property.NodeScope
+    );
+
+    public static final String CACHE_FILE_NAME = "shared_snapshot_cache";
+
     private final ClusterService clusterService;
 
     private final IndexNameExpressionResolver indexNameExpressionResolver;

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/NativeStorageProvider.java

@@ -134,13 +134,8 @@ public ByteSizeValue getMinLocalStorageAvailable() {
         return minLocalStorageAvailable;
     }
 
+    // non-static indirection to enable mocking in tests
     long getUsableSpace(Path path) throws IOException {
-        long freeSpaceInBytes = Environment.getFileStore(path).getUsableSpace();
-
-        /* See: https://bugs.openjdk.java.net/browse/JDK-8162520 */
-        if (freeSpaceInBytes < 0) {
-            freeSpaceInBytes = Long.MAX_VALUE;
-        }
-        return freeSpaceInBytes;
+        return Environment.getUsableSpace(path);
     }
 }

x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/BaseSearchableSnapshotsIntegTestCase.java

@@ -23,6 +23,7 @@
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase;
+import org.elasticsearch.snapshots.SnapshotsService;
 import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotAction;
 import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotRequest;
 import org.elasticsearch.xpack.searchablesnapshots.cache.CacheService;
@@ -72,22 +73,22 @@ protected Settings nodeSettings(int nodeOrdinal) {
             );
         }
         builder.put(
-            FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(),
+            SnapshotsService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(),
             rarely()
                 ? randomBoolean()
                     ? new ByteSizeValue(randomIntBetween(0, 10), ByteSizeUnit.KB)
                     : new ByteSizeValue(randomIntBetween(0, 1000), ByteSizeUnit.BYTES)
                 : new ByteSizeValue(randomIntBetween(1, 10), ByteSizeUnit.MB)
         );
         builder.put(
-            FrozenCacheService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(),
+            SnapshotsService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(),
             rarely()
                 ? new ByteSizeValue(randomIntBetween(4, 1024), ByteSizeUnit.KB)
                 : new ByteSizeValue(randomIntBetween(1, 10), ByteSizeUnit.MB)
         );
         if (randomBoolean()) {
             builder.put(
-                FrozenCacheService.FROZEN_CACHE_RANGE_SIZE_SETTING.getKey(),
+                SnapshotsService.SHARED_CACHE_RANGE_SIZE_SETTING.getKey(),
                 rarely()
                     ? new ByteSizeValue(randomIntBetween(4, 1024), ByteSizeUnit.KB)
                     : new ByteSizeValue(randomIntBetween(1, 10), ByteSizeUnit.MB)

x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java

@@ -57,6 +57,7 @@
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestHandler;
 import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.snapshots.SnapshotsService;
 import org.elasticsearch.snapshots.SourceOnlySnapshotRepository;
 import org.elasticsearch.threadpool.ExecutorBuilder;
 import org.elasticsearch.threadpool.ScalingExecutorBuilder;
@@ -250,9 +251,9 @@ public List<Setting<?>> getSettings() {
             CacheService.SNAPSHOT_CACHE_MAX_FILES_TO_SYNC_AT_ONCE_SETTING,
             CacheService.SNAPSHOT_CACHE_SYNC_SHUTDOWN_TIMEOUT,
             SearchableSnapshotEnableAllocationDecider.SEARCHABLE_SNAPSHOTS_ALLOCATE_ON_ROLLING_RESTART,
-            FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING,
-            FrozenCacheService.SNAPSHOT_CACHE_REGION_SIZE_SETTING,
-            FrozenCacheService.FROZEN_CACHE_RANGE_SIZE_SETTING,
+            SnapshotsService.SNAPSHOT_CACHE_SIZE_SETTING,
+            SnapshotsService.SNAPSHOT_CACHE_REGION_SIZE_SETTING,
+            SnapshotsService.SHARED_CACHE_RANGE_SIZE_SETTING,
             FrozenCacheService.FROZEN_CACHE_RECOVERY_RANGE_SIZE_SETTING,
             FrozenCacheService.SNAPSHOT_CACHE_MAX_FREQ_SETTING,
             FrozenCacheService.SNAPSHOT_CACHE_DECAY_INTERVAL_SETTING,