Merge branch 'main' into jamiemage/strong-type-registry-client

bin/dry-run.rb

@@ -142,6 +142,17 @@
   }
 end
 
+unless ENV["LOCAL_AZURE_ACCESS_TOKEN"].to_s.strip.empty?
+  raise "LOCAL_AZURE_ACCESS_TOKEN supplied without LOCAL_AZURE_FEED_URL" unless ENV["LOCAL_AZURE_FEED_URL"]
+
+  $options[:credentials] << {
+    "type" => "nuget_feed",
+    "host" => "pkgs.dev.azure.com",
+    "url" => ENV.fetch("LOCAL_AZURE_FEED_URL", nil),
+    "token" => ":#{ENV.fetch('LOCAL_AZURE_ACCESS_TOKEN', nil)}"
+  }
+end
+
 unless ENV["LOCAL_CONFIG_VARIABLES"].to_s.strip.empty?
   # For example:
   # "[{\"type\":\"npm_registry\",\"registry\":\
@@ -391,8 +402,8 @@ def fetch_files(fetcher)
     else
       puts "=> cloning into #{$repo_contents_path}"
       FileUtils.rm_rf($repo_contents_path)
+      fetcher.clone_repo_contents
     end
-    fetcher.clone_repo_contents
     if $options[:commit]
       Dir.chdir($repo_contents_path) do
         puts "=> checking out commit #{$options[:commit]}"

nuget/lib/dependabot/nuget/cache_manager.rb

@@ -13,6 +13,8 @@ def self.caching_disabled?
       end
 
       def self.cache(name)
+        return {} if caching_disabled?
+
         @cache ||= {}
         @cache[name] ||= {}
         @cache[name]

nuget/lib/dependabot/nuget/file_fetcher.rb

@@ -3,6 +3,7 @@
 
 require "dependabot/file_fetchers"
 require "dependabot/file_fetchers/base"
+require "dependabot/nuget/cache_manager"
 require "set"
 require "sorbet-runtime"
 
@@ -283,27 +284,34 @@ def imported_property_files
       end
 
       def fetch_imported_property_files(file:, previously_fetched_files:)
-        paths =
-          ImportPathsFinder.new(project_file: file).import_paths +
-          ImportPathsFinder.new(project_file: file).project_reference_paths +
-          ImportPathsFinder.new(project_file: file).project_file_paths
-
-        paths.flat_map do |path|
-          next if previously_fetched_files.map(&:name).include?(path)
-          next if file.name == path
-          next if path.include?("$(")
-
-          fetched_file = fetch_file_from_host(path)
-          grandchild_property_files = fetch_imported_property_files(
-            file: fetched_file,
-            previously_fetched_files: previously_fetched_files + [file]
-          )
-          [fetched_file, *grandchild_property_files]
-        rescue Dependabot::DependencyFileNotFound
-          # Don't worry about missing files too much for now (at least
-          # until we start resolving properties)
-          nil
-        end.compact
+        file_id = file.directory + "/" + file.name
+        @fetched_files ||= {}
+        if @fetched_files[file_id]
+          @fetched_files[file_id]
+        else
+          paths =
+            ImportPathsFinder.new(project_file: file).import_paths +
+            ImportPathsFinder.new(project_file: file).project_reference_paths +
+            ImportPathsFinder.new(project_file: file).project_file_paths
+
+          paths.flat_map do |path|
+            next if previously_fetched_files.map(&:name).include?(path)
+            next if file.name == path
+            next if path.include?("$(")
+
+            fetched_file = fetch_file_from_host(path)
+            grandchild_property_files = fetch_imported_property_files(
+              file: fetched_file,
+              previously_fetched_files: previously_fetched_files + [file]
+            )
+            @fetched_files[file_id] = [fetched_file, *grandchild_property_files]
+            @fetched_files[file_id]
+          rescue Dependabot::DependencyFileNotFound
+            # Don't worry about missing files too much for now (at least
+            # until we start resolving properties)
+            nil
+          end.compact
+        end
       end
     end
   end

nuget/lib/dependabot/nuget/file_parser/packages_config_parser.rb

@@ -26,16 +26,10 @@ def initialize(packages_config:)
         end
 
         def dependency_set
-          return parse_dependencies if CacheManager.caching_disabled?
-
           key = "#{packages_config.name.downcase}::#{packages_config.content.hash}"
           cache = PackagesConfigParser.dependency_set_cache
 
           cache[key] ||= parse_dependencies
-
-          dependency_set = Dependabot::FileParsers::Base::DependencySet.new
-          dependency_set += cache[key]
-          dependency_set
         end
 
         private

nuget/lib/dependabot/nuget/file_parser/project_file_parser.rb

@@ -7,13 +7,13 @@
 require "dependabot/nuget/file_parser"
 require "dependabot/nuget/update_checker"
 require "dependabot/nuget/cache_manager"
+require "dependabot/nuget/nuget_client"
 
 # For details on how dotnet handles version constraints, see:
 # https://docs.microsoft.com/en-us/nuget/reference/package-versioning
 module Dependabot
   module Nuget
     class FileParser
-      # rubocop:disable Metrics/ClassLength
       class ProjectFileParser
         require "dependabot/file_parsers/base/dependency_set"
         require_relative "property_value_finder"
@@ -50,16 +50,10 @@ def initialize(dependency_files:, credentials:)
         end
 
         def dependency_set(project_file:)
-          return parse_dependencies(project_file) if CacheManager.caching_disabled?
-
           key = "#{project_file.name.downcase}::#{project_file.content.hash}"
           cache = ProjectFileParser.dependency_set_cache
 
           cache[key] ||= parse_dependencies(project_file)
-
-          dependency_set = Dependabot::FileParsers::Base::DependencySet.new
-          dependency_set += cache[key]
-          dependency_set
         end
 
         def target_frameworks(project_file:)
@@ -310,16 +304,9 @@ def dependency_url_has_matching_result?(dependency_name, dependency_url)
         end
 
         def dependency_url_has_matching_result_v3?(dependency_name, dependency_url)
-          url = dependency_url.fetch(:search_url)
-          auth_header = dependency_url.fetch(:auth_header)
-          response = execute_search_for_dependency_url(url, auth_header)
-          return false unless response.status == 200
-
-          body = JSON.parse(response.body)
-          data = body["data"]
-          return false unless data.length.positive?
+          versions = NugetClient.get_package_versions_v3(dependency_name, dependency_url)
 
-          data.any? { |result| result["id"].casecmp?(dependency_name) }
+          versions != nil
         end
 
         def dependency_url_has_matching_result_v2?(dependency_name, dependency_url)
@@ -501,7 +488,6 @@ def dotnet_tools_json
           dependency_files.find { |f| f.name.casecmp(".config/dotnet-tools.json").zero? }
         end
       end
-      # rubocop:enable Metrics/ClassLength
     end
   end
 end

nuget/lib/dependabot/nuget/nuget_client.rb

@@ -0,0 +1,99 @@
+# typed: true
+# frozen_string_literal: true
+
+require "dependabot/nuget/cache_manager"
+require "dependabot/nuget/update_checker/repository_finder"
+
+module Dependabot
+  module Nuget
+    class NugetClient
+      def self.get_package_versions_v3(dependency_name, repository_details)
+        # Use the registration URL if possible because it is fast and correct
+        if repository_details[:registration_url]
+          get_versions_from_registration_v3(repository_details)
+        # use the search API if not because it is slow but correct
+        elsif repository_details[:search_url]
+          get_versions_from_search_url_v3(repository_details, dependency_name)
+        # Otherwise, use the versions URL (fast but wrong because it includes unlisted versions)
+        elsif repository_details[:versions_url]
+          get_versions_from_versions_url_v3(repository_details)
+        end
+      end
+
+      private_class_method def self.get_versions_from_versions_url_v3(repository_details)
+        body = execute_search_for_dependency_url(repository_details[:versions_url], repository_details)
+        body&.fetch("versions")
+      end
+
+      private_class_method def self.get_versions_from_registration_v3(repository_details)
+        url = repository_details[:registration_url]
+        body = execute_search_for_dependency_url(url, repository_details)
+
+        return unless body
+
+        pages = body.fetch("items")
+        versions = Set.new
+        pages.each do |page|
+          items = page["items"]
+          if items
+            # inlined entries
+            items.each do |item|
+              catalog_entry = item["catalogEntry"]
+              if catalog_entry["listed"] == true
+                vers = catalog_entry["version"]
+                versions << vers
+              end
+            end
+          else
+            # paged entries
+            page_url = page["@id"]
+            page_body = execute_search_for_dependency_url(page_url, repository_details)
+            items = page_body.fetch("items")
+            items.each do |item|
+              catalog_entry = item.fetch("catalogEntry")
+              versions << catalog_entry.fetch("version") if catalog_entry["listed"] == true
+            end
+          end
+        end
+
+        versions
+      end
+
+      private_class_method def self.get_versions_from_search_url_v3(repository_details, dependency_name)
+        search_url = repository_details[:search_url]
+        body = execute_search_for_dependency_url(search_url, repository_details)
+
+        body&.fetch("data")
+            &.find { |d| d.fetch("id").casecmp(dependency_name.downcase).zero? }
+            &.fetch("versions")
+            &.map { |d| d.fetch("version") }
+      end
+
+      private_class_method def self.execute_search_for_dependency_url(url, repository_details)
+        cache = CacheManager.cache("dependency_url_search_cache")
+        cache[url] ||= Dependabot::RegistryClient.get(
+          url: url,
+          headers: repository_details[:auth_header]
+        )
+
+        response = cache[url]
+
+        return unless response.status == 200
+
+        body = remove_wrapping_zero_width_chars(response.body)
+        JSON.parse(body)
+      rescue Excon::Error::Timeout, Excon::Error::Socket
+        repo_url = repository_details[:repository_url]
+        raise if repo_url == Dependabot::Nuget::UpdateChecker::RepositoryFinder::DEFAULT_REPOSITORY_URL
+
+        raise PrivateSourceTimedOut, repo_url
+      end
+
+      private_class_method def self.remove_wrapping_zero_width_chars(string)
+        string.force_encoding("UTF-8").encode
+              .gsub(/\A[\u200B-\u200D\uFEFF]/, "")
+              .gsub(/[\u200B-\u200D\uFEFF]\Z/, "")
+      end
+    end
+  end
+end

nuget/lib/dependabot/nuget/update_checker/compatibility_checker.rb

@@ -1,11 +1,11 @@
 # typed: true
 # frozen_string_literal: true
 
-require "dependabot/nuget/update_checker"
+require "dependabot/update_checkers/base"
 
 module Dependabot
   module Nuget
-    class UpdateChecker
+    class UpdateChecker < Dependabot::UpdateCheckers::Base
       class CompatibilityChecker
         require_relative "nuspec_fetcher"
         require_relative "nupkg_fetcher"

nuget/lib/dependabot/nuget/update_checker/dependency_finder.rb

@@ -4,12 +4,12 @@
 require "nokogiri"
 require "zip"
 require "stringio"
-require "dependabot/nuget/update_checker"
+require "dependabot/update_checkers/base"
 require "dependabot/nuget/version"
 
 module Dependabot
   module Nuget
-    class UpdateChecker
+    class UpdateChecker < Dependabot::UpdateCheckers::Base
       class DependencyFinder
         require_relative "requirements_updater"
         require_relative "nuspec_fetcher"

nuget/lib/dependabot/nuget/update_checker/property_updater.rb

@@ -1,12 +1,12 @@
 # typed: true
 # frozen_string_literal: true
 
+require "dependabot/update_checkers/base"
 require "dependabot/nuget/file_parser"
-require "dependabot/nuget/update_checker"
 
 module Dependabot
   module Nuget
-    class UpdateChecker
+    class UpdateChecker < Dependabot::UpdateCheckers::Base
       class PropertyUpdater
         require_relative "version_finder"
         require_relative "requirements_updater"