downloads: alternate archive layouts

NEWS.md

@@ -9,6 +9,10 @@
   `Repository`. Recent Bioconductor package `DESCRIPTION` files include the
   `Repository` field, which caused Packrat to believe that those packages
   originated from a CRAN-like repository, not Bioconductor. (#729)
+- Attempt package downloads from multiple package repository archive layouts.
+  Paths used by CRAN-like, Nexus, and older Artifactory repositories are
+  supported. (#583)
+- Attempt URL downloads fewer times before giving up.
 
 # Packrat 0.9.2
 

R/downloader.R

@@ -146,7 +146,7 @@ downloadFile <- function(url,
 # Attempt download.packages multiple times.
 #
 # Assumes we are downloading a single package.
-downloadPackagesWithRetries <- function(name, destdir, repos, type, maxTries = 5L) {
+downloadPackagesWithRetries <- function(name, destdir, repos, type, maxTries = 2L) {
   maxTries <- as.integer(maxTries)
   stopifnot(maxTries > 0L)
   stopifnot(length(name) > 0L)
@@ -173,7 +173,7 @@ downloadPackagesWithRetries <- function(name, destdir, repos, type, maxTries = 5
 #   marker is not `TRUE` after `maxTries` attempts.
 # - Returns `TRUE` if its success marker is `TRUE`.
 # - Passes `destfile` down to `download` via `...`.
-downloadWithRetries <- function(url, ..., maxTries = 5L) {
+downloadWithRetries <- function(url, ..., maxTries = 2L) {
   maxTries <- as.integer(maxTries)
   stopifnot(maxTries > 0L)
 

R/restore.R

@@ -197,33 +197,15 @@ getSourceForPkgRecord <- function(pkgRecord,
       # The version requested is not the version on CRAN; it may be an
       # older version. Look for the older version in the CRAN archive for
       # each named repository.
-      foundVersion <- FALSE
-      for (repo in repos) {
-        tryCatch({
-          archiveUrl <- file.path(repo, "src/contrib/Archive",
-                                  pkgRecord$name,
-                                  pkgSrcFile)
-          downloadWithRetries(archiveUrl,
-                              destfile = file.path(pkgSrcDir, pkgSrcFile),
-                              mode = "wb", quiet = TRUE)
-          foundVersion <- TRUE
-          type <- paste(type, "archived")
-          break
-        }, error = function(e) {
-          # Ignore error and try the next repository
-        })
-      }
+      destFile <- file.path(pkgSrcDir, pkgSrcFile)
+      foundVersion <- downloadArchiveWithRetries(repos, pkgRecord, destFile)
       if (!foundVersion) {
         message("FAILED")
-        stopMsg <- sprintf("Failed to retrieve package sources for %s %s from CRAN (internet connectivity issue?)",
-                           pkgRecord$name,
-                           pkgRecord$version)
-
-        if (!is.na(currentVersion))
-          stopMsg <- paste(stopMsg, sprintf("[%s is current]", currentVersion))
-
-        stop(stopMsg)
+        stop(sprintf("Failed to retrieve package sources for %s %s from a package repository archive (internet connectivity issue?)",
+                     pkgRecord$name,
+                     pkgRecord$version))
       }
+      type <- paste(type, "archived")
     }
   } else if (identical(pkgRecord$source, "github")) {
     archiveUrl <- githubArchiveUrl(pkgRecord)
@@ -342,6 +324,51 @@ getSourceForPkgRecord <- function(pkgRecord,
   }
 }
 
+# Returns TRUE when the package is downloaded, FALSE otherwise.
+downloadArchiveWithRetries <- function(repos, pkgRecord, destFile) {
+  pkgSrcFile <- pkgSrcFilename(pkgRecord)
+
+  # Many repositories follow the CRAN structure, but not all. Try the CRAN
+  # structure across all configured repositories before attempting a
+  # different path.
+  #
+  # The renv implementation probes the repository to determine which
+  # formatter is appropriate. It previously attempted each possibility.
+  # https://github.com/rstudio/renv/blob/731463ec6a52588c148741d2e60c9107c339d941/R/retrieve.R#L951-L1021
+  formatters <- c(
+    # default CRAN format.
+    function(repo) {
+      file.path(repo, "src/contrib/Archive", pkgRecord$name, pkgSrcFile)
+    },
+
+    # Artifactory (old versions / configurations)
+    function(repo) {
+      file.path(repo, "src/contrib/Archive", pkgRecord$name, pkgRecord$version, pkgSrcFile)
+    },
+
+    # Nexus
+    function(repo) {
+      file.path(repo, "src/contrib", pkgSrcFile)
+    }
+  )
+
+  foundVersion <- FALSE
+  for (formatter in formatters) {
+    for (repo in repos) {
+      archiveUrl <- formatter(repo)
+      tryCatch({
+        downloadWithRetries(archiveUrl,
+                            destfile = destFile,
+                            mode = "wb", quiet = TRUE)
+        return(TRUE)
+      }, error = function(e) {
+        # ignore error and try next repo/archive path.
+      })
+    }
+  }
+  return(FALSE)
+}
+
 snapshotSources <- function(project, repos, pkgRecords) {
 
   # Don't snapshot packages included in external.packages