Merge pull request #264 from GEOS-ESM/feature/mathomp4/add-partial-clones

Add support for partial clones

Author: mathomp4

.github/workflows/mepo.yaml

@@ -8,14 +8,14 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest]
-        python-version: ['3.9', '3.10', '3.11', 'pypy-3.9']
+        python-version: ['3.9', '3.10', '3.11', '3.12', 'pypy-3.9', 'pypy-3.10']
 
     name: Python ${{ matrix.python-version }} on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
         cache: 'pip'

CHANGELOG.md

@@ -15,6 +15,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Removed
 
+## [1.52.0] - 2024-01-10
+
+### Added
+
+- Added new `--partial` option to `mepo clone` with two settings: `blobless` and `treeless`. If you set, `--partial=blobless` then
+  the clone will not download blobs by using `--filter=blob:none`. If you set `--partial=treeless` then the clone will not download
+  trees by using `--filter=tree:0`. The `blobless` option is useful for large repos that have a lot of binary files that you don't
+  need. The `treeless` option is even more aggressive and *SHOULD NOT* be used unless you know what you are doing.
+- Add a new section for `.mepoconfig` to allow users to set `--partial` as a default for `mepo clone`.
+
 ## [1.51.1] - 2023-08-25
 
 ### Fixed

etc/mepoconfig-example

@@ -2,7 +2,7 @@
 #
 # .mepoconfig is a config file a la gitconfig with sections and options.
 #
-# Currently, .mepoconfig files recognize two sections: [init] and [alias]
+# Currently, .mepoconfig files recognize three sections: [init], [alias], and [clone].
 #
 # =======================================================================
 #
@@ -24,6 +24,10 @@
 #
 #     mepo clone --style postfix
 #
+#   You set these options by running:
+#
+#     mepo config set init.style <value>
+#
 # =======================================================================
 #
 # [alias] Section
@@ -40,3 +44,40 @@
 #         you can only alias mepo primary commands and not "subcommands" or
 #         "options". So you can have an alias for "commit" and for "branch",
 #         but you can't do an option for "commit -m" or "branch create".
+#
+#   You can set an alias by running:
+#
+#     mepo config set alias.<alias> <command>
+#
+# =======================================================================
+#
+# [clone] Section
+#
+#   The clone section currently recognizes one option, partial.
+#   This has two allowed values: blobless and treeless
+#
+#   So if you have:
+#
+#     [clone]
+#     partial = blobless
+#
+#   This is equivalent to doing:
+#
+#     mepo clone --partial=blobless
+#
+#   which corresponds to the git clone option --filter=blob:none
+#
+#   and similarly for treeless:
+#
+#     [clone]
+#     partial = treeless
+#
+#   is equivalent to doing:
+#
+#     mepo clone --partial=treeless
+#
+#   which corresponds to the git clone option --filter=tree:0
+#
+#   You set these options by running:
+#
+#     mepo config set clone.partial <value>

mepo.d/cmdline/parser.py

@@ -104,6 +104,13 @@ def __clone(self):
             '--allrepos',
             action = 'store_true',
             help = 'Must be passed with -b/--branch. When set, it not only checkouts out the branch/tag for the fixture, but for all the subrepositories as well.')
+        clone.add_argument(
+            '--partial',
+            metavar = 'partial-type',
+            nargs = '?',
+            default = None,
+            choices = ['blobless','treeless'],
+            help = 'Style of partial clone, default: None, allowed options: %(choices)s. Note that blobless means cloning with --filter=blob:none and treeless means cloning with --filter=tree:0. NOTE: We do *not* recommend using "treeless" as it is very aggressive and will cause problems with many git commands.')
 
     def __list(self):
         listcomps = self.subparsers.add_parser(

mepo.d/command/clone/clone.py

@@ -1,7 +1,7 @@
 from state.state    import MepoState, StateDoesNotExistError
 from repository.git import GitRepository
 from command.init   import init as mepo_init
-from utilities      import shellcmd, colors
+from utilities      import shellcmd, colors, mepoconfig
 from urllib.parse   import urlparse
 
 import os
@@ -18,6 +18,19 @@ def run(args):
     if args.allrepos and not args.branch:
         raise RuntimeError("The allrepos option must be used with a branch/tag.")
 
+    # We can get the blobless and treeless options from the config or the args
+    if args.partial:
+        partial = args.partial
+    elif mepoconfig.has_option('clone','partial'):
+        allowed = ['blobless','treeless']
+        partial = mepoconfig.get('clone','partial')
+        if partial not in allowed:
+            raise Exception(f'Detected partial clone type [{partial}] from .mepoconfig is not an allowed partial clone type: {allowed}')
+        else:
+            print(f'Found partial clone type [{partial}] in .mepoconfig')
+    else:
+        partial = None
+
     # If you pass in a config, with clone, it could be outside the repo.
     # So use the full path
     passed_in_config = False
@@ -34,15 +47,15 @@ def run(args):
         last_url_node = p.path.rsplit('/')[-1]
         url_suffix = pathlib.Path(last_url_node).suffix
         if args.directory:
-            local_clone(args.repo_url,args.branch,args.directory)
+            local_clone(args.repo_url,args.branch,args.directory,partial)
             os.chdir(args.directory)
         else:
             if url_suffix == '.git':
                 git_url_directory = pathlib.Path(last_url_node).stem
             else:
                 git_url_directory = last_url_node
 
-            local_clone(args.repo_url,args.branch,git_url_directory)
+            local_clone(args.repo_url,args.branch,git_url_directory,partial)
             os.chdir(git_url_directory)
 
     # Copy the new file into the repo only if we pass it in
@@ -71,7 +84,7 @@ def run(args):
             recurse = comp.recurse_submodules
             # We need the type to handle hashes in components.yaml
             type = comp.version.type
-            git.clone(version,recurse,type,comp.name)
+            git.clone(version,recurse,type,comp.name,partial)
             if comp.sparse:
                 git.sparsify(comp.sparse)
             print_clone_info(comp, max_namelen)
@@ -89,8 +102,12 @@ def print_clone_info(comp, name_width):
     ver_name_type = '({}) {}'.format(comp.version.type, comp.version.name)
     print('{:<{width}} | {:<s}'.format(comp.name, ver_name_type, width = name_width))
 
-def local_clone(url,branch=None,directory=None):
+def local_clone(url,branch=None,directory=None,partial=None):
     cmd1 = 'git clone '
+    if partial == 'blobless':
+        cmd1 += '--filter=blob:none '
+    elif partial == 'treeless':
+        cmd1 += '--filter=tree:0 '
     if branch:
         cmd1 += '--branch {} '.format(branch)
     cmd1 += '--quiet {}'.format(url)

mepo.d/repository/git.py

@@ -39,8 +39,14 @@ def get_full_local_path(self):
     def get_remote_url(self):
         return self.__remote
 
-    def clone(self, version, recurse, type, comp_name):
+    def clone(self, version, recurse, type, comp_name, partial=None):
         cmd1 = 'git clone '
+
+        if partial == 'blobless':
+            cmd1 += '--filter=blob:none '
+        elif partial == 'treeless':
+            cmd1 += '--filter=tree:0 '
+
         if recurse:
             cmd1 += '--recurse-submodules '
 

mepo.d/utest/test_mepo_commands.py

@@ -53,6 +53,7 @@ def setUpClass(cls):
         args.repo_url = None
         args.branch = None
         args.directory = None
+        args.partial = 'blobless'
         mepo_clone.run(args)
         # In order to better test compare, we need to do *something*
         args.comp_name = ['env','cmake','fvdycore']