use history cache when getting last revision of a file

opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/Configuration.java

@@ -205,7 +205,7 @@ public final class Configuration {
      * Set to false if we want to disable fetching history of individual files
      * (by running appropriate SCM command) when the history is not found
      * in history cache for repositories capable of fetching history for
-     * directories. This option affects file based history cache only.
+     * directories.
      */
     private boolean fetchHistoryWhenNotInCache;
     /*

opengrok-indexer/src/main/java/org/opengrok/indexer/history/FileHistoryCache.java

@@ -18,7 +18,7 @@
  */
 
 /*
- * Copyright (c) 2008, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2022, Oracle and/or its affiliates. All rights reserved.
  * Portions Copyright (c) 2018, 2020, Chris Fraire <cfraire@me.com>.
  */
 package org.opengrok.indexer.history;
@@ -198,8 +198,7 @@ public boolean supportsRepository(Repository repository) {
      * @param file the file to find the cache for
      * @return file that might contain cached history for <code>file</code>
      */
-    private static File getCachedFile(File file) throws HistoryException,
-            ForbiddenSymlinkException {
+    private static File getCachedFile(File file) throws HistoryException, ForbiddenSymlinkException {
 
         StringBuilder sb = new StringBuilder();
         sb.append(env.getDataRootPath());
@@ -639,6 +638,14 @@ private void createDirectoriesForFiles(Set<String> files, Repository repository,
     @Override
     public History get(File file, Repository repository, boolean withFiles)
             throws HistoryException, ForbiddenSymlinkException {
+
+        return get(file, repository, withFiles, env.isFetchHistoryWhenNotInCache());
+    }
+
+    @Override
+    public History get(File file, Repository repository, boolean withFiles, boolean fallback)
+            throws HistoryException, ForbiddenSymlinkException {
+
         File cacheFile = getCachedFile(file);
         if (isUpToDate(file, cacheFile)) {
             try {
@@ -662,9 +669,8 @@ public History get(File file, Repository repository, boolean withFiles)
          * since the history of all files in this repository should have been
          * fetched in the first phase of indexing.
          */
-        if (isHistoryIndexDone() && repository.isHistoryEnabled() &&
-                repository.hasHistoryForDirectories() &&
-                !env.isFetchHistoryWhenNotInCache()) {
+        if (isHistoryIndexDone() && repository.isHistoryEnabled() && repository.hasHistoryForDirectories() &&
+                !fallback) {
             return null;
         }
 

opengrok-indexer/src/main/java/org/opengrok/indexer/history/GitRepository.java

@@ -469,7 +469,7 @@ public void accept(String sinceRevision, Consumer<String> visitor) throws Histor
     @Override
     public HistoryEntry getLastHistoryEntry(File file, boolean ui) throws HistoryException {
         History hist = getHistory(file, null, null, 1);
-        return getLastHistoryEntry(hist);
+        return hist.getLastHistoryEntry();
     }
 
     public History getHistory(File file, String sinceRevision, String tillRevision) throws HistoryException {

opengrok-indexer/src/main/java/org/opengrok/indexer/history/History.java

@@ -23,6 +23,8 @@
  */
 package org.opengrok.indexer.history;
 
+import org.jetbrains.annotations.Nullable;
+
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -179,6 +181,18 @@ public void strip() {
         tags.clear();
     }
 
+    /**
+     * @return last (newest) history entry or null
+     */
+    public @Nullable HistoryEntry getLastHistoryEntry() {
+        List<HistoryEntry> historyEntries = getHistoryEntries();
+        if (historyEntries == null || historyEntries.isEmpty()) {
+            return null;
+        }
+
+        return historyEntries.get(0);
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) {

opengrok-indexer/src/main/java/org/opengrok/indexer/history/HistoryCache.java

@@ -18,13 +18,15 @@
  */
 
 /*
- * Copyright (c) 2006, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2006, 2022, Oracle and/or its affiliates. All rights reserved.
  */
 package org.opengrok.indexer.history;
 
 import java.io.File;
 import java.util.Date;
 import java.util.Map;
+
+import org.jetbrains.annotations.Nullable;
 import org.opengrok.indexer.util.ForbiddenSymlinkException;
 
 interface HistoryCache {
@@ -51,17 +53,23 @@ interface HistoryCache {
      * parsing the history information in the repository.
      *
      * @param file The file to retrieve history for
-     * @param repository The external repository to read the history from (can
-     * be <code>null</code>)
-     * @param withFiles A flag saying whether or not the returned history
-     * should include a list of files touched by each changeset. If false,
-     * the implementation is allowed to skip the file list, but it doesn't
-     * have to.
+     * @param repository The external repository to read the history from (can be <code>null</code>)
+     * @param withFiles A flag saying whether the returned history should include a list of files
+     *                  touched by each changeset. If false, the implementation is allowed to skip the file list,
+     *                  but it doesn't have to.
+     * @param fallback whether to fall back to {@link Repository#getHistory(File)}
+     *                 if the history cannot be retrieved from the cache
      * @throws HistoryException if the history cannot be fetched
      * @throws ForbiddenSymlinkException if symbolic-link checking encounters
      * an ineligible link
      */
-    History get(File file, Repository repository, boolean withFiles)
+    History get(File file, @Nullable Repository repository, boolean withFiles, boolean fallback)
+            throws HistoryException, ForbiddenSymlinkException;
+
+    /**
+     * Usually a wrapper of {@link HistoryCache#get(File, Repository, boolean, boolean)}.
+     */
+    History get(File file, @Nullable Repository repository, boolean withFiles)
             throws HistoryException, ForbiddenSymlinkException;
 
     /**

opengrok-indexer/src/main/java/org/opengrok/indexer/history/HistoryGuru.java

@@ -18,7 +18,7 @@
  */
 
 /*
- * Copyright (c) 2005, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2022, Oracle and/or its affiliates. All rights reserved.
  * Portions Copyright (c) 2017, 2020, Chris Fraire <cfraire@me.com>.
  */
 package org.opengrok.indexer.history;
@@ -46,6 +46,7 @@
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
 
+import org.jetbrains.annotations.Nullable;
 import org.opengrok.indexer.configuration.CommandTimeoutType;
 import org.opengrok.indexer.configuration.Configuration.RemoteSCM;
 import org.opengrok.indexer.configuration.PathAccepter;
@@ -155,6 +156,7 @@ public String getCacheInfo() throws HistoryException {
      * <code>HistoryParser</code> does not support annotation
      * @throws IOException if I/O exception occurs
      */
+    @Nullable
     public Annotation annotate(File file, String rev) throws IOException {
         Annotation annotation = null;
 
@@ -228,47 +230,89 @@ public History getHistoryUI(File file) throws HistoryException {
         return getHistory(file, true, true);
     }
 
-    public HistoryEntry getLastHistoryEntry(File file, boolean ui) throws HistoryException {
-        final Repository repo = getRepository(file);
-        if (repo != null) {
-            return repo.getLastHistoryEntry(file, ui);
+    boolean isRepoHistoryEligible(Repository repo, File file, boolean ui) {
+        RemoteSCM rscm = env.getRemoteScmSupported();
+        boolean doRemote = (ui && (rscm == RemoteSCM.UIONLY))
+                || (rscm == RemoteSCM.ON)
+                || (ui || ((rscm == RemoteSCM.DIRBASED) && (repo != null) && repo.hasHistoryForDirectories()));
+
+        return (repo != null && repo.isHistoryEnabled() && repo.isWorking() && repo.fileHasHistory(file)
+                && (!repo.isRemote() || doRemote));
+    }
+
+    @Nullable
+    private History getHistoryFromCache(File file, Repository repository, boolean withFiles, boolean fallback)
+            throws HistoryException, ForbiddenSymlinkException {
+
+        if (useCache() && historyCache.supportsRepository(repository)) {
+            return historyCache.get(file, repository, withFiles, fallback);
         }
+
         return null;
     }
 
     /**
-     * Get the history for the specified file.
+     * @param file file to get the history entry for
+     * @param ui is the request coming from the UI
+     * @return last (newest) history entry for given file or null
+     * @throws HistoryException if history retrieval failed
+     */
+    @Nullable
+    public HistoryEntry getLastHistoryEntry(File file, boolean ui) throws HistoryException {
+        final File dir = file.isDirectory() ? file : file.getParentFile();
+        final Repository repository = getRepository(dir);
+
+        if (!isRepoHistoryEligible(repository, file, ui)) {
+            return null;
+        }
+
+        History history;
+        try {
+            history = getHistoryFromCache(file, repository, false, false);
+            if (history != null) {
+                HistoryEntry lastHistoryEntry = history.getLastHistoryEntry();
+                if (lastHistoryEntry != null) {
+                    LOGGER.log(Level.FINEST, "got latest history entry {0} for ''{1}'' from history cache",
+                            new Object[]{lastHistoryEntry, file});
+                    return lastHistoryEntry;
+                }
+            }
+        } catch (ForbiddenSymlinkException e) {
+            LOGGER.log(Level.FINER, e.getMessage());
+            return null;
+        }
+
+        return repository.getLastHistoryEntry(file, ui);
+    }
+
+    /**
+     * Get the history for the specified file. The history cache is tried first, then the repository.
      *
      * @param file the file to get the history for
-     * @param withFiles whether or not the returned history should contain a
-     * list of files touched by each changeset (the file list may be skipped if
-     * false, but it doesn't have to)
+     * @param withFiles whether the returned history should contain a
+     * list of files touched by each changeset (the file list may be skipped if false, but it doesn't have to)
      * @param ui called from the webapp
      * @return history for the file
      * @throws HistoryException on error when accessing the history
      */
-    public History getHistory(File file, boolean withFiles, boolean ui)
-            throws HistoryException {
-        final File dir = file.isDirectory() ? file : file.getParentFile();
-        final Repository repo = getRepository(dir);
+    public History getHistory(File file, boolean withFiles, boolean ui) throws HistoryException {
 
-        RemoteSCM rscm = env.getRemoteScmSupported();
-        boolean doRemote = (ui && (rscm == RemoteSCM.UIONLY))
-                || (rscm == RemoteSCM.ON)
-                || (ui || ((rscm == RemoteSCM.DIRBASED) && (repo != null) && repo.hasHistoryForDirectories()));
+        final File dir = file.isDirectory() ? file : file.getParentFile();
+        final Repository repository = getRepository(dir);
 
-        if (repo != null && repo.isHistoryEnabled() && repo.isWorking() && repo.fileHasHistory(file)
-                && (!repo.isRemote() || doRemote)) {
+        if (!isRepoHistoryEligible(repository, file, ui)) {
+            return null;
+        }
 
-            if (useCache() && historyCache.supportsRepository(repo)) {
-                try {
-                    return historyCache.get(file, repo, withFiles);
-                } catch (ForbiddenSymlinkException ex) {
-                    LOGGER.log(Level.FINER, ex.getMessage());
-                    return null;
-                }
+        History history;
+        try {
+            history = getHistoryFromCache(file, repository, withFiles, true);
+            if (history != null) {
+                return history;
             }
-            return repo.getHistory(file);
+        } catch (ForbiddenSymlinkException e) {
+            LOGGER.log(Level.FINER, e.getMessage());
+            return null;
         }
 
         return null;
@@ -732,8 +776,9 @@ private List<Repository> getReposFromString(Collection<String> repositories) {
         return repos;
     }
 
-    protected Repository getRepository(File file) {
-        return repositoryLookup.getRepository(file.toPath(), repositoryRoots.keySet(), repositories, PathUtils::getRelativeToCanonical);
+    Repository getRepository(File file) {
+        return repositoryLookup.getRepository(file.toPath(), repositoryRoots.keySet(), repositories,
+                PathUtils::getRelativeToCanonical);
     }
 
     /**

opengrok-indexer/src/main/java/org/opengrok/indexer/history/MercurialRepository.java

@@ -571,9 +571,10 @@ public void accept(String sinceRevision, Consumer<String> visitor) throws Histor
     @Override
     public HistoryEntry getLastHistoryEntry(File file, boolean ui) throws HistoryException {
         History hist = getHistory(file, null, null, 1);
-        return getLastHistoryEntry(hist);
+        return hist.getLastHistoryEntry();
     }
 
+    @Override
     History getHistory(File file, String sinceRevision) throws HistoryException {
         return getHistory(file, sinceRevision, null);
     }

opengrok-indexer/src/main/java/org/opengrok/indexer/history/Repository.java

@@ -18,7 +18,7 @@
  */
 
 /*
- * Copyright (c) 2008, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2008, 2022, Oracle and/or its affiliates. All rights reserved.
  * Portions Copyright (c) 2017, 2020, Chris Fraire <cfraire@me.com>.
  */
 package org.opengrok.indexer.history;
@@ -50,8 +50,6 @@
 import org.opengrok.indexer.util.BufferSink;
 import org.opengrok.indexer.util.Executor;
 
-import org.jetbrains.annotations.NotNull;
-
 /**
  * An interface for an external repository.
  *
@@ -96,7 +94,6 @@ public abstract class Repository extends RepositoryInfo {
      *
      * @return {@code true} if the repository can get history for directories
      */
-    @NotNull
     abstract boolean hasHistoryForDirectories();
 
     public String toString() {
@@ -132,19 +129,6 @@ public String toString() {
      */
     abstract History getHistory(File file) throws HistoryException;
 
-    HistoryEntry getLastHistoryEntry(History hist) {
-        if (hist == null) {
-            return null;
-        }
-
-        List<HistoryEntry> hlist = hist.getHistoryEntries();
-        if (hlist == null || hlist.isEmpty()) {
-            return null;
-        }
-
-        return hlist.get(0);
-    }
-
     /**
      * This is generic implementation that retrieves the full history of given file
      * and returns the latest history entry. This is obviously very inefficient, both in terms of memory and I/O.
@@ -154,15 +138,19 @@ HistoryEntry getLastHistoryEntry(History hist) {
      * @throws HistoryException on error
      */
     public HistoryEntry getLastHistoryEntry(File file, boolean ui) throws HistoryException {
-        History hist;
+        History history;
         try {
-            hist = HistoryGuru.getInstance().getHistory(file, false, ui);
+            history = HistoryGuru.getInstance().getHistory(file, false, ui);
         } catch (HistoryException ex) {
             LOGGER.log(Level.WARNING, "failed to get history for {0}", file);
             return null;
         }
 
-        return getLastHistoryEntry(hist);
+        if (history != null) {
+            return history.getLastHistoryEntry();
+        } else {
+            return null;
+        }
     }
 
     public Repository() {
@@ -519,7 +507,7 @@ public List<String> getIgnoredDirs() {
     /**
      * Determine and return the current version of the repository.
      *
-     * This operation is consider "heavy" so this function should not be
+     * This operation is considered "heavy" so this function should not be
      * called on every web request.
      *
      * @param cmdType command timeout type