Merge pull request dependabot#8324 from dependabot/deivid-rodriguez/f…

…etch-pnpm-404

Catch more 404 PNPM fetch errors and raise them as user errors

npm_and_yarn/lib/dependabot/npm_and_yarn/file_parser.rb

@@ -11,6 +11,7 @@
 require "dependabot/npm_and_yarn/native_helpers"
 require "dependabot/npm_and_yarn/version"
 require "dependabot/npm_and_yarn/requirement"
+require "dependabot/npm_and_yarn/registry_parser"
 require "dependabot/git_metadata_fetcher"
 require "dependabot/git_commit_checker"
 require "dependabot/errors"
@@ -269,7 +270,10 @@ def source_for(name, requirement, lockfile_details)
         return unless resolved_url.start_with?("http")
         return if resolved_url.match?(/(?<!pkg\.)github/)
 
-        registry_source_for(resolved_url, name)
+        RegistryParser.new(
+          resolved_url: resolved_url,
+          credentials: credentials
+        ).registry_source_for(name)
       end
 
       def requirement_for(requirement)
@@ -302,54 +306,6 @@ def git_source_for(requirement)
         }
       end
 
-      def registry_source_for(resolved_url, name)
-        url =
-          if resolved_url.include?("/~/")
-            # Gemfury format
-            resolved_url.split("/~/").first
-          elsif resolved_url.include?("/#{name}/-/#{name}")
-            # MyGet / Bintray format
-            resolved_url.split("/#{name}/-/#{name}").first
-                        .gsub("dl.bintray.com//", "api.bintray.com/npm/").
-              # GitLab format
-              gsub(%r{\/projects\/\d+}, "")
-          elsif resolved_url.include?("/#{name}/-/#{name.split('/').last}")
-            # Sonatype Nexus / Artifactory JFrog format
-            resolved_url.split("/#{name}/-/#{name.split('/').last}").first
-          elsif (cred_url = url_for_relevant_cred(resolved_url)) then cred_url
-          else
-            resolved_url.split("/")[0..2].join("/")
-          end
-
-        { type: "registry", url: url }
-      end
-
-      def url_for_relevant_cred(resolved_url)
-        resolved_url_host = URI(resolved_url).host
-
-        credential_matching_url =
-          credentials
-          .select { |cred| cred["type"] == "npm_registry" }
-          .sort_by { |cred| cred["registry"].length }
-          .find do |details|
-            next true if resolved_url_host == details["registry"]
-
-            uri = if details["registry"]&.include?("://")
-                    URI(details["registry"])
-                  else
-                    URI("https://#{details['registry']}")
-                  end
-            resolved_url_host == uri.host && resolved_url.include?(details["registry"])
-          end
-
-        return unless credential_matching_url
-
-        # Trim the resolved URL so that it ends at the same point as the
-        # credential registry
-        reg = credential_matching_url["registry"]
-        resolved_url.gsub(/#{Regexp.quote(reg)}.*/, "") + reg
-      end
-
       def support_package_files
         @support_package_files ||= sub_package_files.select(&:support_file?)
       end

npm_and_yarn/lib/dependabot/npm_and_yarn/file_parser/json_lock.rb

@@ -7,7 +7,7 @@
 
 module Dependabot
   module NpmAndYarn
-    class FileParser
+    class FileParser < Dependabot::FileParsers::Base
       class JsonLock
         def initialize(dependency_file)
           @dependency_file = dependency_file

npm_and_yarn/lib/dependabot/npm_and_yarn/file_parser/lockfile_parser.rb

@@ -7,7 +7,7 @@
 
 module Dependabot
   module NpmAndYarn
-    class FileParser
+    class FileParser < Dependabot::FileParsers::Base
       class LockfileParser
         require "dependabot/npm_and_yarn/file_parser/yarn_lock"
         require "dependabot/npm_and_yarn/file_parser/pnpm_lock"

npm_and_yarn/lib/dependabot/npm_and_yarn/file_parser/pnpm_lock.rb

@@ -5,7 +5,7 @@
 
 module Dependabot
   module NpmAndYarn
-    class FileParser
+    class FileParser < Dependabot::FileParsers::Base
       class PnpmLock
         def initialize(dependency_file)
           @dependency_file = dependency_file

npm_and_yarn/lib/dependabot/npm_and_yarn/file_parser/yarn_lock.rb

@@ -7,7 +7,7 @@
 
 module Dependabot
   module NpmAndYarn
-    class FileParser
+    class FileParser < Dependabot::FileParsers::Base
       class YarnLock
         def initialize(dependency_file)
           @dependency_file = dependency_file

npm_and_yarn/lib/dependabot/npm_and_yarn/file_updater/npm_lockfile_updater.rb

@@ -13,7 +13,7 @@
 # rubocop:disable Metrics/ClassLength
 module Dependabot
   module NpmAndYarn
-    class FileUpdater
+    class FileUpdater < Dependabot::FileUpdaters::Base
       class NpmLockfileUpdater
         require_relative "npmrc_builder"
         require_relative "package_json_updater"

npm_and_yarn/lib/dependabot/npm_and_yarn/file_updater/npmrc_builder.rb

@@ -5,7 +5,7 @@
 
 module Dependabot
   module NpmAndYarn
-    class FileUpdater
+    class FileUpdater < Dependabot::FileUpdaters::Base
       # Build a .npmrc file from the lockfile content, credentials, and any
       # committed .npmrc
       # We should refactor this to use UpdateChecker::RegistryFinder

npm_and_yarn/lib/dependabot/npm_and_yarn/file_updater/package_json_preparer.rb

@@ -6,7 +6,7 @@
 
 module Dependabot
   module NpmAndYarn
-    class FileUpdater
+    class FileUpdater < Dependabot::FileUpdaters::Base
       class PackageJsonPreparer
         def initialize(package_json_content:)
           @package_json_content = package_json_content

npm_and_yarn/lib/dependabot/npm_and_yarn/file_updater/package_json_updater.rb

@@ -5,7 +5,7 @@
 
 module Dependabot
   module NpmAndYarn
-    class FileUpdater
+    class FileUpdater < Dependabot::FileUpdaters::Base
       class PackageJsonUpdater
         def initialize(package_json:, dependencies:)
           @package_json = package_json

npm_and_yarn/lib/dependabot/npm_and_yarn/file_updater/pnpm_lockfile_updater.rb

@@ -3,11 +3,12 @@
 
 require "dependabot/npm_and_yarn/helpers"
 require "dependabot/npm_and_yarn/update_checker/registry_finder"
+require "dependabot/npm_and_yarn/registry_parser"
 require "dependabot/shared_helpers"
 
 module Dependabot
   module NpmAndYarn
-    class FileUpdater
+    class FileUpdater < Dependabot::FileUpdaters::Base
       class PnpmLockfileUpdater
         require_relative "npmrc_builder"
         require_relative "package_json_updater"
@@ -35,7 +36,7 @@ def updated_pnpm_lock_content(pnpm_lock)
 
         IRRESOLVABLE_PACKAGE = "ERR_PNPM_NO_MATCHING_VERSION"
         INVALID_REQUIREMENT = "ERR_PNPM_SPEC_NOT_SUPPORTED_BY_ANY_RESOLVER"
-        MISSING_PACKAGE = /(?<package_req>.*?) is not in the npm registry, or you have no permission to fetch it/
+        MISSING_PACKAGE = /ERR_PNPM_FETCH_404  GET (?<dependency_url>.*): Not Found - 404/
 
         def run_pnpm_update(pnpm_lock:)
           SharedHelpers.in_a_temporary_repo_directory(base_dir, repo_contents_path) do
@@ -89,9 +90,10 @@ def handle_pnpm_lock_updater_error(error, pnpm_lock)
 
           raise unless error_message.match?(MISSING_PACKAGE)
 
-          package_name = error_message.match(MISSING_PACKAGE)
-                                      .named_captures["package_req"]
-                                      .split(/(?<=\w)\@/).first
+          dependency_url = error_message.match(MISSING_PACKAGE)
+                                        .named_captures["dependency_url"]
+
+          package_name = RegistryParser.new(resolved_url: dependency_url, credentials: credentials).dependency_name
           raise_missing_package_error(package_name, error_message, pnpm_lock)
         end
 

npm_and_yarn/lib/dependabot/npm_and_yarn/file_updater/yarn_lockfile_updater.rb

@@ -14,7 +14,7 @@
 # rubocop:disable Metrics/ClassLength
 module Dependabot
   module NpmAndYarn
-    class FileUpdater
+    class FileUpdater < Dependabot::FileUpdaters::Base
       class YarnLockfileUpdater
         require_relative "npmrc_builder"
         require_relative "package_json_updater"

npm_and_yarn/lib/dependabot/npm_and_yarn/registry_parser.rb

@@ -0,0 +1,75 @@
+# typed: false
+# frozen_string_literal: true
+
+module Dependabot
+  module NpmAndYarn
+    class RegistryParser
+      def initialize(resolved_url:, credentials:)
+        @resolved_url = resolved_url
+        @credentials = credentials
+      end
+
+      def registry_source_for(name)
+        url =
+          if resolved_url.include?("/~/")
+            # Gemfury format
+            resolved_url.split("/~/").first
+          elsif resolved_url.include?("/#{name}/-/#{name}")
+            # MyGet / Bintray format
+            resolved_url.split("/#{name}/-/#{name}").first
+                        .gsub("dl.bintray.com//", "api.bintray.com/npm/").
+              # GitLab format
+              gsub(%r{\/projects\/\d+}, "")
+          elsif resolved_url.include?("/#{name}/-/#{name.split('/').last}")
+            # Sonatype Nexus / Artifactory JFrog format
+            resolved_url.split("/#{name}/-/#{name.split('/').last}").first
+          elsif (cred_url = url_for_relevant_cred) then cred_url
+          else
+            resolved_url.split("/")[0..2].join("/")
+          end
+
+        { type: "registry", url: url }
+      end
+
+      def dependency_name
+        url_base = if resolved_url.include?("/-/")
+                     resolved_url.split("/-/").first
+                   else
+                     resolved_url
+                   end
+
+        url_base.split("/")[3..-1].join("/").gsub("%2F", "/")
+      end
+
+      private
+
+      attr_reader :resolved_url, :credentials
+
+      def url_for_relevant_cred
+        resolved_url_host = URI(resolved_url).host
+
+        credential_matching_url =
+          credentials
+          .select { |cred| cred["type"] == "npm_registry" }
+          .sort_by { |cred| cred["registry"].length }
+          .find do |details|
+            next true if resolved_url_host == details["registry"]
+
+            uri = if details["registry"]&.include?("://")
+                    URI(details["registry"])
+                  else
+                    URI("https://#{details['registry']}")
+                  end
+            resolved_url_host == uri.host && resolved_url.include?(details["registry"])
+          end
+
+        return unless credential_matching_url
+
+        # Trim the resolved URL so that it ends at the same point as the
+        # credential registry
+        reg = credential_matching_url["registry"]
+        resolved_url.gsub(/#{Regexp.quote(reg)}.*/, "") + reg
+      end
+    end
+  end
+end

npm_and_yarn/spec/dependabot/npm_and_yarn/file_updater/pnpm_lockfile_updater_spec.rb

@@ -10,7 +10,7 @@
       dependency_files: files,
       dependencies: dependencies,
       credentials: credentials,
-      repo_contents_path: nil
+      repo_contents_path: repo_contents_path
     )
   end
   let(:dependencies) { [dependency] }
@@ -50,19 +50,24 @@
       source: nil
     }]
   end
+
+  let(:files) { project_dependency_files(project_name) }
+
   let(:pnpm_lock) do
     files.find { |f| f.name == "pnpm-lock.yaml" }
   end
 
   let(:tmp_path) { Dependabot::Utils::BUMP_TMP_DIR_PATH }
 
+  let(:repo_contents_path) { build_tmp_repo(project_name, path: "projects") }
+
   before { FileUtils.mkdir_p(tmp_path)  }
 
   subject(:updated_pnpm_lock_content) { updater.updated_pnpm_lock_content(pnpm_lock) }
 
   describe "errors" do
     context "with a dependency version that can't be found" do
-      let(:files) { project_dependency_files("pnpm/yanked_version") }
+      let(:project_name) { "pnpm/yanked_version" }
 
       it "raises a helpful error" do
         expect { updated_pnpm_lock_content }
@@ -71,7 +76,7 @@
     end
 
     context "with an invalid requirement in the package.json" do
-      let(:files) { project_dependency_files("pnpm/invalid_requirement") }
+      let(:project_name) { "pnpm/invalid_requirement" }
 
       it "raises a helpful error" do
         expect { updated_pnpm_lock_content }
@@ -80,7 +85,36 @@
     end
 
     context "with a dependency that can't be found" do
-      let(:files) { project_dependency_files("pnpm/nonexistent_dependency_yanked_version") }
+      let(:project_name) { "pnpm/nonexistent_dependency_yanked_version" }
+
+      it "raises a helpful error" do
+        expect { updated_pnpm_lock_content }
+          .to raise_error(Dependabot::PrivateSourceAuthenticationFailure)
+      end
+    end
+
+    context "with a locked dependency that can't be found" do
+      let(:dependency_name) { "@googleapis/youtube" }
+      let(:version) { "13.0.0" }
+      let(:previous_version) { "10.1.0" }
+      let(:requirements) do
+        [{
+          file: "package.json",
+          requirement: "^13.0.0",
+          groups: ["dependencies"],
+          source: nil
+        }]
+      end
+      let(:previous_requirements) do
+        [{
+          file: "package.json",
+          requirement: "^10.1.0",
+          groups: ["dependencies"],
+          source: nil
+        }]
+      end
+
+      let(:project_name) { "pnpm/nonexistent_locked_dependency" }
 
       it "raises a helpful error" do
         expect { updated_pnpm_lock_content }

npm_and_yarn/spec/fixtures/projects/pnpm/nonexistent_locked_dependency/package.json

@@ -0,0 +1,9 @@
+{
+	"name": "margins",
+	"version": "0.0.1",
+	"type": "module",
+	"dependencies": {
+		"@googleapis/youtube": "^10.1.0",
+                "@tiptap-pro/extension-mathematics": "latest"
+	}
+}