[SR-12851] System-wide cache of SwiftPM dependencies

Sources/SourceControl/GitRepository.swift

@@ -11,6 +11,7 @@
 import TSCBasic
 import Dispatch
 import TSCUtility
+import class Foundation.NSFileManager.FileManager
 
 public struct GitCloneError: Swift.Error, CustomStringConvertible {
 
@@ -34,6 +35,12 @@ public class GitRepositoryProvider: RepositoryProvider {
     /// Reference to process set, if installed.
     private let processSet: ProcessSet?
 
+    /// Initializes a GitRepositoryProvider
+    /// - Parameters:
+    ///   - processSet: Reference to process set.
+    ///   - cachePath: Path to the directory where all cached git repositories are stored. If `nil` is passed as the`cachePath`
+    ///   fetched repositores will not be cached.
+    ///   - maxCacheSize: Maximum size of the cache in bytes.
     public init(processSet: ProcessSet? = nil) {
         self.processSet = processSet
     }
@@ -44,20 +51,17 @@ public class GitRepositoryProvider: RepositoryProvider {
         // NOTE: We intentionally do not create a shallow clone here; the
         // expected cost of iterative updates on a full clone is less than on a
         // shallow clone.
-
         precondition(!localFileSystem.exists(path))
 
         // FIXME: We need infrastructure in this subsystem for reporting
         // status information.
 
-        let process = Process(
-            args: Git.tool, "clone", "--mirror", repository.url, path.pathString, environment: Git.environment)
-        // Add to process set.
+        let process = Process(args: Git.tool, "clone", "--mirror",
+                              repository.url, path.pathString, environment: Git.environment)
         try processSet?.add(process)
 
         try process.launch()
         let result = try process.waitUntilExit()
-
         // Throw if cloning failed.
         guard result.exitStatus == .terminated(code: 0) else {
             throw GitCloneError(
@@ -75,35 +79,13 @@ public class GitRepositoryProvider: RepositoryProvider {
         repository: RepositorySpecifier,
         at sourcePath: AbsolutePath,
         to destinationPath: AbsolutePath,
-        editable: Bool
+        editable: Bool = false
     ) throws {
-
-        if editable {
-            // For editable clones, i.e. the user is expected to directly work on them, first we create
-            // a clone from our cache of repositories and then we replace the remote to the one originally
-            // present in the bare repository.
-            try Process.checkNonZeroExit(args:
-                    Git.tool, "clone", "--no-checkout", sourcePath.pathString, destinationPath.pathString)
-            // The default name of the remote.
-            let origin = "origin"
-            // In destination repo remove the remote which will be pointing to the source repo.
-            let clone = GitRepository(path: destinationPath)
-            // Set the original remote to the new clone.
-            try clone.setURL(remote: origin, url: repository.url)
-            // FIXME: This is unfortunate that we have to fetch to update remote's data.
-            try clone.fetch()
-        } else {
-            // Clone using a shared object store with the canonical copy.
-            //
-            // We currently expect using shared storage here to be safe because we
-            // only ever expect to attempt to use the working copy to materialize a
-            // revision we selected in response to dependency resolution, and if we
-            // re-resolve such that the objects in this repository changed, we would
-            // only ever expect to get back a revision that remains present in the
-            // object storage.
-            try Process.checkNonZeroExit(args:
-                    Git.tool, "clone", "--shared", "--no-checkout", sourcePath.pathString, destinationPath.pathString)
-        }
+        // For editable clones, i.e. the user is expected to directly work on them, first we create
+        // a clone from our cache of repositories and then we replace the remote to the one originally
+        // present in the bare repository.
+        try Process.checkNonZeroExit(args: Git.tool, "clone", "--no-checkout", "--reference", sourcePath.pathString,
+                                     repository.url, "--dissociate", destinationPath.pathString)
     }
 
     public func checkoutExists(at path: AbsolutePath) throws -> Bool {
@@ -277,6 +259,29 @@ public class GitRepository: Repository, WorkingCheckout {
         }
     }
 
+    /// Adds a remote to the repository
+    /// - Parameters:
+    ///   - remote: The name of the remote to add.
+    ///   - url: The url of the remote to add.
+    public func add(remote: String, url: String) throws {
+        try queue.sync {
+            try Process.checkNonZeroExit(
+                args: Git.tool, "-C", path.pathString, "remote", "add", remote, url)
+            return
+        }
+    }
+
+
+    /// Removes a remote from the repository
+    /// - Parameter remote: The name of the remote to remove.
+    public func remove(remote: String) throws {
+        try queue.sync {
+            try Process.checkNonZeroExit(
+                args: Git.tool, "-C", path.pathString, "remote", "remove", remote)
+            return
+        }
+    }
+
     // MARK: Repository Interface
 
     /// Returns the tags present in repository.

Sources/SourceControl/RepositoryManager.swift

@@ -113,14 +113,6 @@ public class RepositoryManager {
             precondition(status == .available, "cloneCheckout() called in invalid state")
             try self.manager.cloneCheckout(self, to: path, editable: editable)
         }
-
-        fileprivate func toJSON() -> JSON {
-            return .init([
-                "status": status.rawValue,
-                "repositoryURL": repository,
-                "subpath": subpath,
-            ])
-        }
     }
 
     /// The path under which repositories are stored.
@@ -132,18 +124,6 @@ public class RepositoryManager {
     /// The delegate interface.
     private let delegate: RepositoryManagerDelegate?
 
-    // FIXME: We should use a more sophisticated map here, which tracks the
-    // full specifier but then is capable of efficiently determining if two
-    // repositories map to the same location.
-    //
-    /// The map of registered repositories.
-    fileprivate var repositories: [String: RepositoryHandle] = [:]
-
-    /// The map of serialized repositories.
-    ///
-    /// NOTE: This is to be used only for persistence support.
-    fileprivate var serializedRepositories: [String: JSON] = [:]
-
     /// Queue to protect concurrent reads and mutations to repositories registery.
     private let serialQueue = DispatchQueue(label: "org.swift.swiftpm.repomanagerqueue-serial")
 
@@ -159,9 +139,6 @@ public class RepositoryManager {
     /// The filesystem to operate on.
     public let fileSystem: FileSystem
 
-    /// Simple persistence helper.
-    private let persistence: SimplePersistence
-
     /// Create a new empty manager.
     ///
     /// - Parameters:
@@ -185,24 +162,6 @@ public class RepositoryManager {
         self.operationQueue = OperationQueue()
         self.operationQueue.name = "org.swift.swiftpm.repomanagerqueue-concurrent"
         self.operationQueue.maxConcurrentOperationCount = 10
-
-        self.persistence = SimplePersistence(
-            fileSystem: fileSystem,
-            schemaVersion: 1,
-            statePath: path.appending(component: "checkouts-state.json"))
-
-        // Load the state from disk, if possible.
-        do {
-            _ = try self.persistence.restoreState(self)
-        } catch {
-            // State restoration errors are ignored, for now.
-            //
-            // FIXME: We need to do something better here.
-            print("warning: unable to restore checkouts state: \(error)")
-
-            // Try to save the empty state.
-            try? self.persistence.saveState(self)
-        }
     }
 
     /// Get a handle to a repository.
@@ -227,6 +186,7 @@ public class RepositoryManager {
             // Dispatch the action we want to take on the serial queue of the handle.
             handle.serialQueue.sync {
                 let result: LookupResult
+                let lock = FileLock(name: repository.fileSystemIdentifier, cachePath: self.path)
 
                 switch handle.status {
                 case .available:
@@ -243,7 +203,9 @@ public class RepositoryManager {
                             self.delegate?.handleWillUpdate(handle: handle)
                         }
 
-                        try repo.fetch()
+                        try lock.withLock {
+                            try repo.fetch()
+                        }
 
                         self.callbacksQueue.async {
                             self.delegate?.handleDidUpdate(handle: handle)
@@ -267,8 +229,10 @@ public class RepositoryManager {
                     // Fetch the repo.
                     var fetchError: Swift.Error? = nil
                     do {
-                        // Start fetching.
-                        try self.provider.fetch(repository: handle.repository, to: repositoryPath)
+                        try lock.withLock {
+                            // Start fetching.
+                            try self.provider.fetch(repository: handle.repository, to: repositoryPath)
+                        }
                         // Update status to available.
                         handle.status = .available
                         result = .success(handle)
@@ -282,21 +246,6 @@ public class RepositoryManager {
                     self.callbacksQueue.async {
                         self.delegate?.fetchingDidFinish(handle: handle, error: fetchError)
                     }
-
-                    // Save the manager state.
-                    self.serialQueue.sync {
-                        do {
-                            // Update the serialized repositories map.
-                            //
-                            // We do this so we don't have to read the other
-                            // handles when saving the sate of this handle.
-                            self.serializedRepositories[repository.url] = handle.toJSON()
-                            try self.persistence.saveState(self)
-                        } catch {
-                            // FIXME: Handle failure gracefully, somehow.
-                            fatalError("unable to save manager state \(error)")
-                        }
-                    }
                 }
                 // Call the completion handler.
                 self.callbacksQueue.async {
@@ -311,20 +260,11 @@ public class RepositoryManager {
     /// Note: This method is thread safe.
     private func getHandle(repository: RepositorySpecifier) -> RepositoryHandle {
         return serialQueue.sync {
-
-            // Reset if the state file was deleted during the lifetime of RepositoryManager.
-            if !self.serializedRepositories.isEmpty && !self.persistence.stateFileExists() {
-                self.unsafeReset()
-            }
-
-            let handle: RepositoryHandle
-            if let oldHandle = self.repositories[repository.url] {
-                handle = oldHandle
-            } else {
-                let subpath = RelativePath(repository.fileSystemIdentifier)
-                let newHandle = RepositoryHandle(manager: self, repository: repository, subpath: subpath)
-                self.repositories[repository.url] = newHandle
-                handle = newHandle
+            let subpath = RelativePath(repository.fileSystemIdentifier)
+            let handle = RepositoryHandle(manager: self, repository: repository, subpath: subpath)
+            let repositoryPath = path.appending(RelativePath(repository.fileSystemIdentifier))
+            if fileSystem.exists(repositoryPath) {
+                handle.status = .available
             }
             return handle
         }
@@ -342,25 +282,30 @@ public class RepositoryManager {
         to destinationPath: AbsolutePath,
         editable: Bool
     ) throws {
-        try provider.cloneCheckout(
-            repository: handle.repository,
-            at: path.appending(handle.subpath),
-            to: destinationPath,
-            editable: editable)
+        let lock = FileLock(name: handle.repository.basename, cachePath: self.path)
+        // FIXME: Workaround for `FileLock` only working on `LocaFileSystem`
+        if localFileSystem.exists(self.path) {
+            try lock.withLock {
+                try provider.cloneCheckout(
+                    repository: handle.repository,
+                    at: path.appending(handle.subpath),
+                    to: destinationPath,
+                    editable: editable)
+            }
+        } else {
+            try provider.cloneCheckout(
+                repository: handle.repository,
+                at: path.appending(handle.subpath),
+                to: destinationPath,
+                editable: editable)
+        }
     }
 
     /// Removes the repository.
     public func remove(repository: RepositorySpecifier) throws {
         try serialQueue.sync {
-            // If repository isn't present, we're done.
-            guard let handle = repositories[repository.url] else {
-                return
-            }
-            repositories[repository.url] = nil
-            serializedRepositories[repository.url] = nil
-            let repositoryPath = path.appending(handle.subpath)
+            let repositoryPath = path.appending(RelativePath(repository.fileSystemIdentifier))
             try fileSystem.removeFileTree(repositoryPath)
-            try self.persistence.saveState(self)
         }
     }
 
@@ -375,31 +320,10 @@ public class RepositoryManager {
 
     /// Performs the reset operation without the serial queue.
     private func unsafeReset() {
-        self.repositories = [:]
-        self.serializedRepositories = [:]
         try? self.fileSystem.removeFileTree(path)
     }
 }
 
-// MARK: Persistence
-extension RepositoryManager: SimplePersistanceProtocol {
-
-    public func restore(from json: JSON) throws {
-        // Update the serialized repositories.
-        //
-        // We will use this to save the state so we don't have to read the other
-        // handles when saving the sate of a handle.
-        self.serializedRepositories = try json.get("repositories")
-        self.repositories = try serializedRepositories.mapValues({
-            try RepositoryHandle(manager: self, json: $0)
-        })
-    }
-
-    public func toJSON() -> JSON {
-        return JSON(["repositories": JSON(self.serializedRepositories)])
-    }
-}
-
 extension RepositoryManager.RepositoryHandle: CustomStringConvertible {
     public var description: String {
         return "<\(type(of: self)) subpath:\(subpath)>"

Sources/Workspace/Workspace.swift

@@ -15,6 +15,7 @@ import PackageLoading
 import PackageModel
 import PackageGraph
 import SourceControl
+import class Foundation.NSFileManager.FileManager
 
 /// Enumeration of the different reasons for which the resolver needs to be run.
 public enum WorkspaceResolveReason: Equatable {
@@ -453,7 +454,12 @@ public class Workspace {
         self.resolvedFile = pinsFile
         self.additionalFileRules = additionalFileRules
 
-        let repositoriesPath = self.dataPath.appending(component: "repositories")
+        /// The default location of the git repository cache
+        let repositoriesPath: AbsolutePath = {
+            let cacheURL = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first!
+            return AbsolutePath(cacheURL.path).appending(components: "org.swift.swiftpm", "repositories")
+        }()
+
         let repositoryManager = repositoryManager ?? RepositoryManager(
             path: repositoriesPath,
             provider: repositoryProvider,

Tests/SourceControlTests/RepositoryManagerTests.swift

@@ -217,13 +217,6 @@ class RepositoryManagerTests: XCTestCase {
             // Remove the repo.
             try manager.remove(repository: dummyRepo)
 
-            // Check removing the repo updates the persistent file.
-            do {
-                let checkoutsStateFile = path.appending(component: "checkouts-state.json")
-                let jsonData = try JSON(bytes: localFileSystem.readFileContents(checkoutsStateFile))
-                XCTAssertEqual(jsonData.dictionary?["object"]?.dictionary?["repositories"]?.dictionary?[dummyRepo.url], nil)
-            }
-
             // We should get a new handle now because we deleted the exisiting repository.
             XCTNonNil(prevHandle) {
                 try XCTAssert($0 !== manager.lookupSynchronously(repository: dummyRepo))
@@ -293,7 +286,6 @@ class RepositoryManagerTests: XCTestCase {
             do {
                 let delegate = DummyRepositoryManagerDelegate()
                 var manager = RepositoryManager(path: path, provider: provider, delegate: delegate)
-                try! localFileSystem.removeFileTree(path.appending(component: "checkouts-state.json"))
                 manager = RepositoryManager(path: path, provider: provider, delegate: delegate)
                 let dummyRepo = RepositorySpecifier(url: "dummy")
 
@@ -386,10 +378,6 @@ class RepositoryManagerTests: XCTestCase {
             _ = try manager.lookupSynchronously(repository: dummyRepo)
             XCTAssertEqual(delegate.didFetch.count, 1)
 
-            // Delete the checkout state file.
-            let stateFile = repos.appending(component: "checkouts-state.json")
-            try localFileSystem.removeFileTree(stateFile)
-
             // We should refetch the repository since we lost the state file.
             _ = try manager.lookupSynchronously(repository: dummyRepo)
             XCTAssertEqual(delegate.didFetch.count, 2)