Merge pull request #5661 from dependabot/deivid-rodriguez/standard-py…

…thon

Initial work on standard Python support

python/helpers/lib/parser.py

@@ -11,11 +11,63 @@
     install_req_from_line,
     install_req_from_parsed_requirement,
 )
+
+from packaging.requirements import InvalidRequirement, Requirement
+import toml
+
 # Inspired by pips internal check:
 # https://github.com/pypa/pip/blob/0bb3ac87f5bb149bd75cceac000844128b574385/src/pip/_internal/req/req_file.py#L35
 COMMENT_RE = re.compile(r'(^|\s+)#.*$')
 
 
+def parse_pep621_dependencies(pyproject_path):
+    project_toml = toml.load(pyproject_path)['project']
+
+    def parse_toml_section_pep621_dependencies(pyproject_path, dependencies):
+        requirement_packages = []
+
+        def version_from_req(specifier_set):
+            if (len(specifier_set) == 1 and
+                    next(iter(specifier_set)).operator in {"==", "==="}):
+                return next(iter(specifier_set)).version
+
+        for dependency in dependencies:
+            try:
+                req = Requirement(dependency)
+            except InvalidRequirement as e:
+                print(json.dumps({"error": repr(e)}))
+                exit(1)
+            else:
+                requirement_packages.append({
+                    "name": req.name,
+                    "version": version_from_req(req.specifier),
+                    "markers": str(req.marker) or None,
+                    "file": pyproject_path,
+                    "requirement": str(req.specifier),
+                    "extras": sorted(list(req.extras))
+                })
+
+        return requirement_packages
+
+    dependencies = parse_toml_section_pep621_dependencies(
+        pyproject_path,
+        project_toml['dependencies']
+    )
+
+    if 'optional-dependencies' in project_toml:
+        optional_dependencies_toml = project_toml['optional-dependencies']
+
+        for group in optional_dependencies_toml:
+            group_dependencies = parse_toml_section_pep621_dependencies(
+                pyproject_path,
+                optional_dependencies_toml[group]
+            )
+
+            dependencies.extend(group_dependencies)
+
+    return json.dumps({"result": dependencies})
+
+
 def parse_requirements(directory):
     # Parse the requirements.txt
     requirement_packages = []

python/helpers/run.py

@@ -10,6 +10,8 @@
         print(parser.parse_requirements(args["args"][0]))
     elif args["function"] == "parse_setup":
         print(parser.parse_setup(args["args"][0]))
+    elif args["function"] == "parse_pep621_dependencies":
+        print(parser.parse_pep621_dependencies(args["args"][0]))
     elif args["function"] == "get_dependency_hash":
         print(hasher.get_dependency_hash(*args["args"]))
     elif args["function"] == "get_pipfile_hash":

python/lib/dependabot/python/file_fetcher.rb

@@ -5,7 +5,7 @@
 require "dependabot/file_fetchers"
 require "dependabot/file_fetchers/base"
 require "dependabot/python/requirement_parser"
-require "dependabot/python/file_parser/poetry_files_parser"
+require "dependabot/python/file_parser/pyproject_files_parser"
 require "dependabot/errors"
 
 module Dependabot
@@ -24,7 +24,7 @@ def self.required_files_in?(filenames)
         # If this repo is using a Pipfile return true
         return true if filenames.include?("Pipfile")
 
-        # If this repo is using Poetry return true
+        # If this repo is using pyproject.toml return true
         return true if filenames.include?("pyproject.toml")
 
         return true if filenames.include?("setup.py")
@@ -69,7 +69,7 @@ def pipenv_files
       end
 
       def pyproject_files
-        [pyproject, pyproject_lock, poetry_lock].compact
+        [pyproject, pyproject_lock, poetry_lock, pdm_lock].compact
       end
 
       def requirement_files
@@ -141,6 +141,10 @@ def poetry_lock
         @poetry_lock ||= fetch_file_if_present("poetry.lock")
       end
 
+      def pdm_lock
+        @pdm_lock ||= fetch_file_if_present("pdm.lock")
+      end
+
       def requirements_txt_files
         req_txt_and_in_files.select { |f| f.name.end_with?(".txt") }
       end
@@ -296,8 +300,8 @@ def fetch_path_setup_file(path, allow_pyproject: false)
               fetch_submodules: true
             ).tap { |f| f.support_file = true }
           rescue Dependabot::DependencyFileNotFound
-            # For Poetry projects attempt to fetch a pyproject.toml at the
-            # given path instead of a setup.py. We do not require a
+            # For projects with pyproject.toml attempt to fetch a pyproject.toml
+            # at the given path instead of a setup.py. We do not require a
             # setup.py to be present, so if none can be found, simply return
             return [] unless allow_pyproject
 
@@ -395,7 +399,7 @@ def poetry_path_setup_file_paths
         return [] unless pyproject
 
         paths = []
-        Dependabot::Python::FileParser::PoetryFilesParser::POETRY_DEPENDENCY_TYPES.each do |dep_type|
+        Dependabot::Python::FileParser::PyprojectFilesParser::POETRY_DEPENDENCY_TYPES.each do |dep_type|
           next unless parsed_pyproject.dig("tool", "poetry", dep_type)
 
           parsed_pyproject.dig("tool", "poetry", dep_type).each do |_, req|

python/lib/dependabot/python/file_parser.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "toml-rb"
 require "dependabot/dependency"
 require "dependabot/file_parsers"
 require "dependabot/file_parsers/base"
@@ -15,11 +14,9 @@ module Dependabot
   module Python
     class FileParser < Dependabot::FileParsers::Base
       require_relative "file_parser/pipfile_files_parser"
-      require_relative "file_parser/poetry_files_parser"
+      require_relative "file_parser/pyproject_files_parser"
       require_relative "file_parser/setup_file_parser"
 
-      POETRY_DEPENDENCY_TYPES =
-        %w(tool.poetry.dependencies tool.poetry.dev-dependencies).freeze
       DEPENDENCY_GROUP_KEYS = [
         {
           pipfile: "packages",
@@ -42,7 +39,7 @@ def parse
         dependency_set = DependencySet.new
 
         dependency_set += pipenv_dependencies if pipfile
-        dependency_set += poetry_dependencies if using_poetry?
+        dependency_set += pyproject_file_dependencies if pyproject
         dependency_set += requirement_dependencies if requirement_files.any?
         dependency_set += setup_file_dependencies if setup_file || setup_cfg_file
 
@@ -62,9 +59,9 @@ def pipenv_dependencies
           dependency_set
       end
 
-      def poetry_dependencies
-        @poetry_dependencies ||=
-          PoetryFilesParser.
+      def pyproject_file_dependencies
+        @pyproject_file_dependencies ||=
+          PyprojectFilesParser.
           new(dependency_files: dependency_files).
           dependency_set
       end
@@ -105,18 +102,6 @@ def group_from_filename(filename)
         end
       end
 
-      def included_in_pipenv_deps?(dep_name)
-        return false unless pipfile
-
-        pipenv_dependencies.dependencies.map(&:name).include?(dep_name)
-      end
-
-      def included_in_poetry_deps?(dep_name)
-        return false unless using_poetry?
-
-        poetry_dependencies.dependencies.map(&:name).include?(dep_name)
-      end
-
       def blocking_marker?(dep)
         return false if dep["markers"] == "None"
         return true if dep["markers"].include?("<")
@@ -215,15 +200,6 @@ def pipfile_lock
         @pipfile_lock ||= get_original_file("Pipfile.lock")
       end
 
-      def using_poetry?
-        return false unless pyproject
-        return true if poetry_lock || pyproject_lock
-
-        !TomlRB.parse(pyproject.content).dig("tool", "poetry").nil?
-      rescue TomlRB::ParseError, TomlRB::ValueOverwriteError
-        raise Dependabot::DependencyFileNotParseable, pyproject.path
-      end
-
       def output_file_regex(filename)
         "--output-file[=\s]+#{Regexp.escape(filename)}(?:\s|$)"
       end

python/lib/dependabot/python/file_parser/poetry_files_parser.rb → python/lib/dependabot/python/file_parser/pyproject_files_parser.rb

@@ -12,7 +12,7 @@
 module Dependabot
   module Python
     class FileParser
-      class PoetryFilesParser
+      class PyprojectFilesParser
         POETRY_DEPENDENCY_TYPES = %w(dependencies dev-dependencies).freeze
 
         # https://python-poetry.org/docs/dependency-specification/
@@ -25,7 +25,7 @@ def initialize(dependency_files:)
         def dependency_set
           dependency_set = Dependabot::FileParsers::Base::DependencySet.new
 
-          dependency_set += pyproject_dependencies
+          dependency_set += pyproject_dependencies if using_poetry? || using_pep621?
           dependency_set += lockfile_dependencies if lockfile
 
           dependency_set
@@ -36,6 +36,14 @@ def dependency_set
         attr_reader :dependency_files
 
         def pyproject_dependencies
+          if using_poetry?
+            poetry_dependencies
+          else
+            pep621_dependencies
+          end
+        end
+
+        def poetry_dependencies
           dependencies = Dependabot::FileParsers::Base::DependencySet.new
 
           POETRY_DEPENDENCY_TYPES.each do |type|
@@ -59,6 +67,44 @@ def pyproject_dependencies
           dependencies
         end
 
+        def pep621_dependencies
+          dependencies = Dependabot::FileParsers::Base::DependencySet.new
+
+          # PDM is not yet supported, so we want to ignore it for now because in
+          # the current state of things, going on would result in updating
+          # pyproject.toml but leaving pdm.lock out of sync, which is
+          # undesirable. Leave PDM alone until properly supported
+          return dependencies if using_pdm?
+
+          parsed_pep621_dependencies.each do |dep|
+            # If a requirement has a `<` or `<=` marker then updating it is
+            # probably blocked. Ignore it.
+            next if dep["markers"].include?("<")
+
+            # If no requirement, don't add it
+            next if dep["requirement"].empty?
+
+            dependencies <<
+              Dependency.new(
+                name: normalised_name(dep["name"], dep["extras"]),
+                version: dep["version"]&.include?("*") ? nil : dep["version"],
+                requirements: [{
+                  requirement: dep["requirement"],
+                  file: Pathname.new(dep["file"]).cleanpath.to_path,
+                  source: nil,
+                  groups: [dep["requirement_type"]]
+                }],
+                package_manager: "pip"
+              )
+          end
+
+          dependencies
+        end
+
+        def normalised_name(name, extras)
+          NameNormaliser.normalise_including_extras(name, extras)
+        end
+
         # @param req can be an Array, Hash or String that represents the constraints for a dependency
         def parse_requirements_from(req, type)
           [req].flatten.compact.filter_map do |requirement|
@@ -75,6 +121,18 @@ def parse_requirements_from(req, type)
           end
         end
 
+        def using_poetry?
+          !parsed_pyproject.dig("tool", "poetry").nil?
+        end
+
+        def using_pep621?
+          !parsed_pyproject.dig("project", "dependencies").nil?
+        end
+
+        def using_pdm?
+          using_pep621? && pdm_lock
+        end
+
         # Create a DependencySet where each element has no requirement. Any
         # requirements will be added when combining the DependencySet with
         # other DependencySets.
@@ -146,6 +204,24 @@ def lockfile
           poetry_lock || pyproject_lock
         end
 
+        def parsed_pep621_dependencies
+          SharedHelpers.in_a_temporary_directory do
+            write_temporary_pyproject
+
+            SharedHelpers.run_helper_subprocess(
+              command: "pyenv exec python #{NativeHelpers.python_helper_path}",
+              function: "parse_pep621_dependencies",
+              args: [pyproject.name]
+            )
+          end
+        end
+
+        def write_temporary_pyproject
+          path = pyproject.name
+          FileUtils.mkdir_p(Pathname.new(path).dirname)
+          File.write(path, pyproject.content)
+        end
+
         def parsed_lockfile
           return parsed_poetry_lock if poetry_lock
           return parsed_pyproject_lock if pyproject_lock
@@ -160,6 +236,11 @@ def poetry_lock
           @poetry_lock ||=
             dependency_files.find { |f| f.name == "poetry.lock" }
         end
+
+        def pdm_lock
+          @pdm_lock ||=
+            dependency_files.find { |f| f.name == "pdm.lock" }
+        end
       end
     end
   end

python/lib/dependabot/python/file_updater.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "toml-rb"
 require "dependabot/file_updaters"
 require "dependabot/file_updaters/base"
 require "dependabot/shared_helpers"
@@ -61,7 +62,13 @@ def resolver_type
         # Otherwise, this is a top-level dependency, and we can figure out
         # which resolver to use based on the filename of its requirements
         return :pipfile if changed_req_files.any?("Pipfile")
-        return :poetry if changed_req_files.any?("pyproject.toml")
+
+        if changed_req_files.any?("pyproject.toml")
+          return :poetry if poetry_based?
+
+          return :requirements
+        end
+
         return :pip_compile if changed_req_files.any? { |f| f.end_with?(".in") }
 
         :requirements
@@ -119,6 +126,12 @@ def check_required_files
         raise "Missing required files!"
       end
 
+      def poetry_based?
+        return false unless pyproject
+
+        !TomlRB.parse(pyproject.content).dig("tool", "poetry").nil?
+      end
+
       def pipfile
         @pipfile ||= get_original_file("Pipfile")
       end

python/lib/dependabot/python/file_updater/poetry_file_updater.rb

@@ -131,7 +131,7 @@ def freeze_dependencies_being_updated(pyproject_content)
         end
 
         def lock_declaration_to_new_version!(poetry_object, dep)
-          Dependabot::Python::FileParser::PoetryFilesParser::POETRY_DEPENDENCY_TYPES.each do |type|
+          Dependabot::Python::FileParser::PyprojectFilesParser::POETRY_DEPENDENCY_TYPES.each do |type|
             names = poetry_object[type]&.keys || []
             pkg_name = names.find { |nm| normalise(nm) == dep.name }
             next unless pkg_name

python/lib/dependabot/python/file_updater/pyproject_preparer.rb

@@ -52,7 +52,7 @@ def freeze_top_level_dependencies_except(dependencies)
           poetry_object = pyproject_object["tool"]["poetry"]
           excluded_names = dependencies.map(&:name) + ["python"]
 
-          Dependabot::Python::FileParser::PoetryFilesParser::POETRY_DEPENDENCY_TYPES.each do |key|
+          Dependabot::Python::FileParser::PyprojectFilesParser::POETRY_DEPENDENCY_TYPES.each do |key|
             next unless poetry_object[key]
 
             source_types = %w(directory file url)