Option whether to pull submodules or not + remember default

ctfoood/admin.py

@@ -63,7 +63,7 @@ class ChalAdmin(admin.ModelAdmin):
                            ('vm_setup', 'owner_user', 'owner_group'),
                            ('public_checkout',),
                            )}),
-        ("Git pull config", {'fields': (('autopull_url','autopull_branch'),),}),
+        ("Git pull config", {'fields': (('autopull_url','autopull_branch','autopull_submodules'),),}),
         ("Git pull deploy key", {'fields': ('autopull_deploy_key',), 'classes': ('collapse',)}),
         ("Game info", {'fields': (('points',),
                                   ('solves_n','solves_url'),

ctfoood/importer.py

@@ -163,17 +163,19 @@ def run_tester_cmd(path:str, arg:str=None, format=None,
     return 0, tester_output, tester_gave_errors, tester_gave_warnings
 
 
-def git_clone(chal: Chal, pull_from:str, pull_branch:str,
+def git_clone(chal: Chal, pull_from:str, pull_branch:str, submodules:bool=True,
         real_terminal=False) -> Tuple[int, str, str, str, Optional[str]]:
     """Return returncode, output, commit_hash, temp dir containing clone, deploy key temp file"""
     clone_output = ""
     destdir = tempfile.mkdtemp(prefix='mio_clone_', suffix="__"+chal.name)
 
-    # Leaving this in do_autopull for now, as it uses those variables later
+    # Note: pull_branch, pull_from, submodules actually have defaults in Chal
+    #       do_autopull handles that for now
 
     assert pull_from
-    cmd = ['git','clone','-q','--depth=1','--no-tags',
-            '--shallow-submodules','--recurse-submodules']
+    cmd = ['git','clone','-q','--depth=1','--no-tags']
+    if submodules:
+        cmd += ['--shallow-submodules','--recurse-submodules']
     # TODO[opt]: --reference-if-able=previous_checkout --dissociate
     if pull_branch:
         cmd += ['-b', pull_branch ]
@@ -222,6 +224,7 @@ def do_autopull(chal: Chal, user: User, run_tester:bool=False,
         real_terminal=False, docker_cleanup=True, make_public:bool=False,
         as_default:bool=False, dockerhub:bool=False,
         tester_log_level:str="WARNING",
+        submodules: Optional[bool]=None,
         just_test_deployed:Optional[VM]=None, just_healthcheck:bool=False) -> Tuple[int, str, Optional[ChalCheckout]]:
     """Returns errcode, output, resulting_checkout_object"""
 
@@ -254,8 +257,13 @@ def do_autopull(chal: Chal, user: User, run_tester:bool=False,
             pull_from = chal.autopull_url
         if pull_branch is None:
             pull_branch = chal.autopull_branch
+        if submodules is None:
+            submodules = chal.autopull_submodules
+        if submodules is None:
+            # If it was never specified, default to yes
+            submodules = True
         returncode, clone_output, commit_hash, destdir, dkf = git_clone(chal=chal,
-                pull_from=pull_from, pull_branch=pull_branch,
+                pull_from=pull_from, pull_branch=pull_branch, submodules=submodules,
                 real_terminal=real_terminal)
         if returncode != 0:
             logger.error("git clone failed (%s) %s %s -- %s", chal, pull_from, pull_branch, clone_output)

ctfoood/models.py

@@ -147,6 +147,7 @@ class Chal(models.Model):
             help_text="Private SSH key -- upload the public one as a deploy key on github")
     autopull_url = models.CharField('autopull URL', blank=True, max_length=300, help_text="Argument to git clone")
     autopull_branch = models.CharField(blank=True, max_length=50, help_text="Override the remote default")
+    autopull_submodules = models.BooleanField(blank=True, null=True, help_text="Whether to pull submodules too. Unspecified currently defaults to yes.")
 
     # Extra fields beyond the info.yml (whose data is captured per-checkout)
     extra_description = models.TextField(blank=True, help_text="markdown, shown below the original description")
@@ -218,6 +219,7 @@ def get_deploy_key_fingerprint(self) -> str:
     def show_git_clone(self) -> str:
         if not self.autopull_url:
             return "# missing autopull_url :("
+        # TODO: submodules if actually present + used
         return f'git clone [...] {self.autopull_url}' + \
                 (f' -b {self.autopull_branch}' if self.autopull_branch else '') + \
                 (("   # With deploy key "+self.get_deploy_key_fingerprint()) if self.autopull_deploy_key else '')
@@ -306,6 +308,7 @@ def get_absolute_url(self):
     def get_tags(self):
         return self.tags.union(self.chal.extra_tags.all()).all()
     def public_git_clone_cmd(self) -> str:
+        # TODO: submodules if actually present + used
         if (self.chal.source_url.startswith('https://github.com/')) and not self.dirty_tree:
             return f'git clone {self.chal.source_url}' + \
                     (f' -b {self.branch}' if self.branch else '')

import.py

@@ -57,6 +57,7 @@ def main():
     advanced.add_argument("--user", help="Username (default: user with the lowest id)")
     advanced.add_argument("--group", help="Group name (default: the user's), only for --create-chal")
     advanced.add_argument("--no-cleanup", action="store_true", help="Would normally cleanup by grepping 'docker ps' and 'docker images' for the challenge name")
+    advanced.add_argument("--submodules", metavar="yes/no", choices=('yes','no'), help="Override the default")
     forcreatechal = parser.add_argument_group('For --create-chal')
     forcreatechal.add_argument("--format", help="Only used with --create-chal (default: reads it from the git url)")
     forcreatechal.add_argument("--accept-unclean", action="store_true", help="Would normally reject name collisions and the like -- but it's fine if we're sure we're importing only one challenge at a time")
@@ -82,6 +83,11 @@ def main():
         assert user.groups, "User {} is not in any group, and right now we need one. Just create a group and add {} to it.".format(user,user)  # TODO: auto-create?
         group = user.groups.order_by('id')[0]
 
+    submodules = args.submodules
+    if submodules is not None:
+        submodules = (submodules == 'yes')
+    assert submodules is None or submodules is True or submodules is False
+
 
 
     chal_already_exists = Chal.objects.filter(name=args.chalname).exists()
@@ -129,6 +135,7 @@ def main():
         chal = Chal.objects.create(name=args.chalname, format=y, type='normal',
                 owner_user=user, owner_group=group, autopull_url=args.pull_from,
                 autopull_branch=args.branch if args.branch else "",
+                autopull_submodules=submodules,
                 source_url=source_url, solves_url=solves_url)
         try:
             chal.full_clean()
@@ -156,7 +163,8 @@ def main():
             is_autopull=False, real_terminal=True,
             tester_log_level=args.log_level,
             docker_cleanup=not args.no_cleanup, make_public=args.public,
-            as_default=args.as_default, dockerhub=args.dockerhub)
+            as_default=args.as_default, dockerhub=args.dockerhub,
+            submodules=submodules)
 
 
     #print(output)