Move the srcToStore cache into fetchToStore()

src/libexpr/eval.cc

@@ -331,7 +331,6 @@ EvalState::EvalState(
     , debugStop(false)
     , trylevel(0)
     , asyncPathWriter(AsyncPathWriter::make(store))
-    , srcToStore(make_ref<decltype(srcToStore)::element_type>())
     , importResolutionCache(make_ref<decltype(importResolutionCache)::element_type>())
     , fileEvalCache(make_ref<decltype(fileEvalCache)::element_type>())
     , positionToDocComment(make_ref<decltype(positionToDocComment)::element_type>())
@@ -2534,23 +2533,16 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
     if (nix::isDerivation(path.path.abs()))
         error<EvalError>("file names are not allowed to end in '%1%'", drvExtension).debugThrow();
 
-    auto dstPathCached = getConcurrent(*srcToStore, path);
-
-    auto dstPath = dstPathCached ? *dstPathCached : [&]() {
-        auto dstPath = fetchToStore(
-            fetchSettings,
-            *store,
-            path.resolveSymlinks(SymlinkResolution::Ancestors),
-            settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy,
-            computeBaseName(path, pos),
-            ContentAddressMethod::Raw::NixArchive,
-            nullptr,
-            repair);
-        allowPath(dstPath);
-        srcToStore->try_emplace(path, dstPath);
-        printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath));
-        return dstPath;
-    }();
+    auto dstPath = fetchToStore(
+        fetchSettings,
+        *store,
+        path.resolveSymlinks(SymlinkResolution::Ancestors),
+        settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy,
+        computeBaseName(path, pos),
+        ContentAddressMethod::Raw::NixArchive,
+        nullptr,
+        repair);
+    allowPath(dstPath);
 
     context.insert(NixStringContextElem::Opaque{.path = dstPath});
     return dstPath;

src/libexpr/include/nix/expr/eval.hh

@@ -464,10 +464,6 @@ public:
 
 private:
 
-    /* Cache for calls to addToStore(); maps source paths to the store
-       paths. */
-    const ref<boost::concurrent_flat_map<SourcePath, StorePath>> srcToStore;
-
     /**
      * A cache that maps paths to "resolved" paths for importing Nix
      * expressions, i.e. `/foo` to `/foo/default.nix`.

src/libfetchers/fetch-to-store.cc

@@ -3,8 +3,23 @@
 #include "nix/fetchers/fetch-settings.hh"
 #include "nix/util/environment-variables.hh"
 
+#include <boost/unordered/concurrent_flat_map.hpp>
+
 namespace nix {
 
+struct SrcToStore
+{
+    boost::concurrent_flat_map<
+        std::tuple<SourcePath, ContentAddressMethod::Raw, std::string>,
+        std::tuple<StorePath, Hash, FetchMode>>
+        cache;
+};
+
+ref<SrcToStore> fetchers::Settings::createSrcToStore()
+{
+    return make_ref<SrcToStore>();
+}
+
 fetchers::Cache::Key
 makeSourcePathToHashCacheKey(std::string_view fingerprint, ContentAddressMethod method, const CanonPath & path)
 {
@@ -36,6 +51,14 @@ std::pair<StorePath, Hash> fetchToStore2(
     PathFilter * filter,
     RepairFlag repair)
 {
+    auto srcToStoreKey = std::make_tuple(path, method.raw, std::string(name));
+
+    if (!filter) {
+        auto dstPathCached = getConcurrent(settings.srcToStore->cache, srcToStoreKey);
+        if (dstPathCached && (mode == FetchMode::DryRun || std::get<2>(*dstPathCached) == FetchMode::Copy))
+            return std::make_pair(std::get<0>(*dstPathCached), std::get<1>(*dstPathCached));
+    }
+
     std::optional<fetchers::Cache::Key> cacheKey;
 
     auto [subpath, fingerprint] = filter ? std::pair<CanonPath, std::optional<std::string>>{path.path, std::nullopt}
@@ -67,7 +90,6 @@ std::pair<StorePath, Hash> fetchToStore2(
         static auto barf = getEnv("_NIX_TEST_BARF_ON_UNCACHEABLE").value_or("") == "1";
         if (barf && !filter && !(path.to_string().starts_with("/") || path.to_string().starts_with("«path:/")))
             throw Error("source path '%s' is uncacheable (filter=%d)", path, (bool) filter);
-        // FIXME: could still provide in-memory caching keyed on `SourcePath`.
         debug("source path '%s' is uncacheable", path);
     }
 
@@ -102,8 +124,9 @@ std::pair<StorePath, Hash> fetchToStore2(
                           throw Error("path '%s' lacks a CA field", store.printStorePath(storePath));
                       info->ca->hash;
                   });
-                  debug(
-                      "copied '%s' to '%s' (hash '%s')",
+                  printMsg(
+                      lvlChatty,
+                      "copied source '%s' -> '%s' (hash '%s')",
                       path,
                       store.printStorePath(storePath),
                       hash.to_string(HashFormat::SRI, true));
@@ -113,6 +136,9 @@ std::pair<StorePath, Hash> fetchToStore2(
     if (cacheKey)
         settings.getCache()->upsert(*cacheKey, {{"hash", hash.to_string(HashFormat::SRI, true)}});
 
+    if (!filter)
+        settings.srcToStore->cache.insert_or_assign(srcToStoreKey, std::make_tuple(storePath, hash, mode));
+
     return {storePath, hash};
 }
 

src/libfetchers/include/nix/fetchers/fetch-settings.hh

@@ -14,8 +14,9 @@
 namespace nix {
 
 struct GitRepo;
+struct SrcToStore;
 
-}
+} // namespace nix
 
 namespace nix::fetchers {
 
@@ -156,6 +157,15 @@ struct Settings : public Config
 
     ref<GitRepo> getTarballCache() const;
 
+    /**
+     * In-memory cache for calls to fetchToStore(); maps source paths to their store
+     * paths / hashes.
+     */
+    static ref<SrcToStore> createSrcToStore();
+
+    const ref<SrcToStore> srcToStore = createSrcToStore();
+
+
 private:
     mutable Sync<std::shared_ptr<Cache>> _cache;