Add an ability to clone/pull only one group using its id or full path or name

README.rst

@@ -60,6 +60,8 @@ Usage
     +---------------+---------------+---------------------------+
     | exclude       | -x            | `GITLABBER_EXCLUDE`       |
     +---------------+---------------+---------------------------+
+    | root_group    | -g            | `GITLABBER_ROOT_GROUP`    |
+    +---------------+---------------+---------------------------+
 
 * To view the tree run the command with your includes/excludes and the `-p` flag it will print your tree like so
 
@@ -114,6 +116,8 @@ Usage
                             comma delimited list of glob patterns of paths to projects or groups to clone/pull
     -x csv, --exclude csv
                             comma delimited list of glob patterns of paths to projects or groups to exclude from clone/pull
+    -g root_group, --root-group root_group
+                            id/full_path/full_name of a group to use as the root instead of the entire gitlab tree
     -r, --recursive       clone/pull git submodules recursively
     --version             print the version
 

gitlabber/cli.py

@@ -38,12 +38,13 @@ def main():
     excludes=split(args.exclude)
 
     tree = GitlabTree(args.url, args.token, args.method, args.naming, args.archived.api_value, includes,
-                      excludes, args.file, args.concurrency, args.recursive, args.verbose)
+                      excludes, args.file, args.concurrency, args.recursive, args.verbose,
+                      args.root_group)
     log.debug("Reading projects tree from gitlab at [%s]", args.url)
     tree.load_tree()
 
     if tree.is_empty():
-        log.fatal("The tree is empty, check your include/exclude patterns or run with more verbosity for debugging")
+        log.fatal("The tree is empty, check your include/exclude patterns and/or a root_group value or run with more verbosity for debugging")
         sys.exit(1) 
 
     if args.print:
@@ -171,6 +172,13 @@ def parse_args(argv=None):
         metavar=('csv'),
         default=os.environ.get('GITLABBER_EXCLUDE', ""),
         help='comma delimited list of glob patterns of paths to projects or groups to exclude from clone/pull')
+    parser.add_argument(
+        '-g',
+        '--root-group',
+        metavar=('root_group'),
+        default=os.environ.get('GITLABBER_ROOT_GROUP'),
+        help='id/full_path/full_name of a group to use as the root instead of the entire gitlab tree',
+        )
     parser.add_argument(
         '-r',
         '--recursive',

gitlabber/gitlab_tree.py

@@ -1,4 +1,5 @@
 from gitlab import Gitlab
+from gitlab.exceptions import GitlabGetError
 from anytree import Node, RenderTree
 from anytree.exporter import DictExporter, JsonExporter
 from anytree.importer import DictImporter
@@ -16,7 +17,8 @@
 
 class GitlabTree:
 
-    def __init__(self, url, token, method, naming, archived=None, includes=[], excludes=[], in_file=None, concurrency=1, recursive=False, disable_progress=False):
+    def __init__(self, url, token, method, naming, archived=None, includes=[], excludes=[], in_file=None, concurrency=1, recursive=False, disable_progress=False,
+                 root_group=None):
         self.includes = includes
         self.excludes = excludes
         self.url = url
@@ -31,6 +33,7 @@ def __init__(self, url, token, method, naming, archived=None, includes=[], exclu
         self.recursive = recursive
         self.disable_progress = disable_progress
         self.progress = ProgressBar('* loading tree', disable_progress)
+        self.root_group = root_group
 
     @staticmethod
     def get_ca_path():
@@ -105,27 +108,47 @@ def get_projects(self, group, parent):
         self.progress.update_progress_length(len(projects))
         self.add_projects(parent, projects)
 
+    def add_groups(self, parent, groups):
+        for group in groups:
+            lazy_group = self.gitlab.groups.get(group.id, lazy=True)
+            group_id = group.name if self.naming == FolderNaming.NAME else group.path
+            node = None
+            if parent is not None:
+                node = self.make_node(group_id, parent, url=group.web_url)
+            elif self.root_group in (group.id, group.full_path, group.full_name):
+                node = self.root = Node("", root_path="", url=group.web_url)
+            self.progress.show_progress(group_id, 'group')
+            self.get_subgroups(lazy_group, node)
+            if node is not None:
+                self.get_projects(lazy_group, node)
+            if node == self.root:
+                break
+
     def get_subgroups(self, group, parent):
         subgroups = group.subgroups.list(as_list=False, archived=self.archived)
         self.progress.update_progress_length(len(subgroups))
-        for subgroup_def in subgroups:
-            subgroup = self.gitlab.groups.get(subgroup_def.id)
-            subgroup_id = subgroup.name if self.naming == FolderNaming.NAME else subgroup.path
-            node = self.make_node(subgroup_id, parent, url=subgroup.web_url)
-            self.progress.show_progress(node.name, 'group')
-            self.get_subgroups(subgroup, node)
-            self.get_projects(subgroup, node)
+        self.add_groups(parent, subgroups)
 
     def load_gitlab_tree(self):
-        groups = self.gitlab.groups.list(as_list=False, archived=self.archived)
+        groups = []
+        if self.root_group is not None:
+            try:
+                # Try to get a group by the given 'root_group' value so we
+                # won't need to traverse a whole group tree. In case of
+                # success, assume the 'root_group' value is equal to the 'id'
+                # or 'full_path' group field value.
+                groups = [self.gitlab.groups.get(self.root_group)]
+            except GitlabGetError as e:
+                if e.response_code == 404:
+                    log.debug("Couldn't find a group with id='%s'" % self.root_group)
+                else:
+                    raise
+        if not groups:
+            groups = self.gitlab.groups.list(as_list=False,
+                                             archived=self.archived,
+                                             top_level_only=True)
         self.progress.init_progress(len(groups))
-        for group in groups:
-            if group.parent_id is None:
-                group_id = group.name if self.naming == FolderNaming.NAME else group.path
-                node = self.make_node(group_id, self.root, url=group.web_url)
-                self.progress.show_progress(node.name, 'group')
-                self.get_subgroups(group, node)
-                self.get_projects(group, node)
+        self.add_groups(self.root if self.root_group is None else None, groups)
 
         elapsed = self.progress.finish_progress()
         log.debug("Loading projects tree from gitlab took [%s]", elapsed)
@@ -161,7 +184,7 @@ def print_tree_native(self):
         for pre, _, node in RenderTree(self.root):
             line = ""
             if node.is_root:
-                line = "%s%s [%s]" % (pre, "root", self.url)
+                line = "%s%s [%s]" % (pre, "root", node.url)
             else:
                 line = "%s%s [%s]" % (pre, node.name, node.root_path)
             print(line)

tests/gitlab_test_utils.py

@@ -2,6 +2,7 @@
 import json
 from unittest import mock
 from gitlabber import gitlab_tree
+from gitlab.exceptions import GitlabGetError
 
 URL = "http://gitlab.my.com/"
 TOKEN = "MOCK_TOKEN"
@@ -23,8 +24,8 @@
 class MockNode:
     def __init__(self, id, name, url, subgroups=mock.MagicMock(), projects=mock.MagicMock(), parent_id=None):
         self.id = id
-        self.name = name
-        self.path = name
+        self.name = self.full_name = name
+        self.path = self.full_path = name
         self.url = url
         self.web_url = url
         self.ssh_url_to_repo = url
@@ -35,24 +36,25 @@ def __init__(self, id, name, url, subgroups=mock.MagicMock(), projects=mock.Magi
 
 
 class Listable:
-    def __init__(self, list_result, get_result=None, archive_result=None):
+    def __init__(self, list_result, get_results=None, archive_result=None):
         self.list_result = list_result
-        self.get_result = get_result
+        self.get_results = get_results
         self.archive_result = archive_result
 
-    def list(self, as_list=False, archived=None):
+    def list(self, as_list=False, archived=None, top_level_only=False):
         if archived is None:
             return [self.list_result, self.archive_result] if self.archive_result is not None else [self.list_result]
         elif archived is True:
             return [self.archive_result]
         else:
             return [self.list_result]
 
-    def get(self, id):
-        if self.get_result is not None:
-            return self.get_result
-        else:
-            return self.list_result
+    def get(self, id, lazy=False):
+        try:
+            return list(filter(lambda n: id in (n.id, n.full_path),
+                               self.get_results)).pop()
+        except IndexError:
+            raise GitlabGetError(response_code=404)
 
 
 def validate_root(root):
@@ -64,15 +66,17 @@ def validate_root(root):
 
 
 def validate_group(group):
-    assert group.name == GROUP_NAME
+    if not group.is_root:
+        assert group.name == GROUP_NAME
     assert group.url == GROUP_URL
     assert group.is_leaf is False
     assert len(group.children) == 1
     assert group.height == 2
 
 
 def validate_subgroup(subgroup):
-    assert subgroup.name == SUBGROUP_NAME
+    if not subgroup.is_root:
+        assert subgroup.name == SUBGROUP_NAME
     assert subgroup.url == SUBGROUP_URL
     assert subgroup.is_leaf is False
     assert len(subgroup.children) == 1
@@ -93,41 +97,70 @@ def validate_tree(root):
     validate_project(root.children[0].children[0].children[0])
 
 
-def create_test_gitlab(monkeypatch, includes=None, excludes=None, in_file=None):
+def fixup_child_nodes(node):
+    for children in filter(lambda c: isinstance(c, Listable), [node.subgroups,
+                                                               node.projects]):
+        for child in filter(lambda r: isinstance(r, MockNode),
+                            [children.list_result, children.archive_result]):
+            child.full_path = '/'.join([node.full_path, child.path])
+            child.full_name = ' / '.join([node.full_name, child.name])
+            fixup_child_nodes(child)
+
+
+def append_node(nodes, *args, **kwargs):
+    node = MockNode(len(nodes), *args, **kwargs)
+    nodes.append(node)
+    fixup_child_nodes(node)
+    return node
+
+
+def create_test_gitlab(monkeypatch, includes=None, excludes=None, in_file=None,
+                       root_group=None):
     gl = gitlab_tree.GitlabTree(
-        URL, TOKEN, "ssh", "name", includes=includes, excludes=excludes, in_file=in_file)
-    projects = Listable(MockNode(2, PROJECT_NAME, PROJECT_URL))
-    subgroup_node = MockNode(2, SUBGROUP_NAME, SUBGROUP_URL, projects=projects)
+        URL, TOKEN, "ssh", "name", includes=includes, excludes=excludes, in_file=in_file,
+        root_group=root_group)
+    nodes = []
+    projects = Listable(append_node(nodes, PROJECT_NAME, PROJECT_URL))
+    subgroup_node = append_node(nodes, SUBGROUP_NAME, SUBGROUP_URL,
+                                projects=projects)
     subgroups = Listable(subgroup_node)
-    groups = Listable(MockNode(2, GROUP_NAME, GROUP_URL,
-                               subgroups=subgroups), subgroup_node)
+    groups = Listable(append_node(nodes, GROUP_NAME, GROUP_URL,
+                                  subgroups=subgroups), nodes)
     monkeypatch.setattr(gl.gitlab, "groups", groups)
     return gl
 
 
 def create_test_gitlab_with_toplevel_subgroups(monkeypatch):
     gl = gitlab_tree.GitlabTree(URL, TOKEN, "ssh", "path")
-    groups = Listable([MockNode(2, GROUP_NAME, GROUP_URL),
-                       MockNode(2, GROUP_NAME, GROUP_URL, parent_id=1)])
+    nodes = []
+    groups = Listable([append_node(nodes, GROUP_NAME, GROUP_URL),
+                       append_node(nodes, GROUP_NAME, GROUP_URL, parent_id=1)],
+                       nodes)
     monkeypatch.setattr(gl.gitlab, "groups", groups)
     return gl
 
 
 def create_test_gitlab_with_archived(monkeypatch, includes=None, excludes=None, in_file=None, archived=None):
     gl = gitlab_tree.GitlabTree(
         URL, TOKEN, "ssh", "name", includes=includes, excludes=excludes, in_file=in_file, archived=archived)
-    project_node = MockNode(1, PROJECT_NAME, PROJECT_URL)
-    archived_project_node = MockNode(
-        2, "_archived_" + PROJECT_NAME, "_archived_" + PROJECT_URL)
+    nodes = []
+    project_node = append_node(nodes, PROJECT_NAME, PROJECT_URL)
+    archived_project_node = append_node(nodes, "_archived_" + PROJECT_NAME,
+                                        "_archived_" + PROJECT_URL)
     projects = Listable(project_node, archive_result=archived_project_node)
-    subgroup_node = MockNode(2, SUBGROUP_NAME, SUBGROUP_URL, projects=projects)
-    archived_subgroup_node = MockNode(
-        2, "_archived_" + SUBGROUP_NAME, "_archived_" + SUBGROUP_URL, projects=projects)
+    subgroup_node = append_node(nodes, SUBGROUP_NAME, SUBGROUP_URL,
+                                projects=projects)
+    archived_subgroup_node = append_node(nodes, "_archived_" + SUBGROUP_NAME,
+                                         "_archived_" + SUBGROUP_URL,
+                                         projects=projects)
     subgroups = Listable(subgroup_node, archive_result=archived_subgroup_node)
-    archived_subgroups = Listable(archived_subgroup_node, archive_result=archived_subgroup_node)
-    group_node = MockNode(2, GROUP_NAME, GROUP_URL, subgroups=archived_subgroups)
-    archived_group_node = MockNode(2, "_archived_" + GROUP_NAME, "_archived_" + GROUP_URL, subgroups=archived_subgroups)
-    groups = Listable(group_node, get_result=subgroup_node, archive_result=archived_group_node)
+    group_node = append_node(nodes, GROUP_NAME, GROUP_URL,
+                             subgroups=subgroups)
+    archived_group_node = append_node(nodes, "_archived_" + GROUP_NAME,
+                                      "_archived_" + GROUP_URL,
+                                      subgroups=subgroups)
+    groups = Listable(group_node, archive_result=archived_group_node,
+                      get_results=nodes)
     monkeypatch.setattr(gl.gitlab, "groups", groups)
     # gl.print_tree()
     return gl

tests/test_cli.py

@@ -36,7 +36,7 @@ def test_args_version():
 def test_args_logging(mock_tree, mock_log, mock_os, mock_sys, mock_logging):
     args_mock = mock.Mock()
     args_mock.return_value = Node(
-        name="test", version=None, verbose=True, include="", exclude="", url="test_url", token="test_token", method=CloneMethod.SSH, naming=FolderNaming.PATH, archived=ArchivedResults.INCLUDE, file=None, concurrency=1, recursive=False, disble_progress=True, print=None, dest=".")
+        name="test", version=None, verbose=True, include="", exclude="", url="test_url", token="test_token", method=CloneMethod.SSH, naming=FolderNaming.PATH, archived=ArchivedResults.INCLUDE, file=None, concurrency=1, recursive=False, disble_progress=True, print=None, dest=".", root_group=None)
     cli.parse_args = args_mock
 
     mock_streamhandler = mock.Mock()
@@ -57,7 +57,7 @@ def test_args_include(mock_tree):
     exc_groups = "/exc**,/exc**"
     args_mock = mock.Mock()
     args_mock.return_value = Node(
-        name="test", version=None, debug=None, include=inc_groups, exclude=exc_groups, url="test_url", token="test_token", method=CloneMethod.SSH, naming=FolderNaming.NAME, archived=ArchivedResults.INCLUDE, file=None, concurrency=1, recursive=False, disble_progress=True, print=None, dest=".")
+        name="test", version=None, debug=None, include=inc_groups, exclude=exc_groups, url="test_url", token="test_token", method=CloneMethod.SSH, naming=FolderNaming.NAME, archived=ArchivedResults.INCLUDE, file=None, concurrency=1, recursive=False, disble_progress=True, print=None, dest=".", root_group=None)
     cli.parse_args = args_mock
 
     split_mock = mock.Mock()
@@ -73,7 +73,7 @@ def test_args_include(mock_tree):
 def test_args_include(mock_tree):
     args_mock = mock.Mock()
     args_mock.return_value = Node(
-        name="test", version=None, verbose=None, include="", exclude="", url="test_url", token="test_token", method=CloneMethod.SSH, naming=FolderNaming.NAME, archived=ArchivedResults.INCLUDE, file=None, concurrency=1, recursive=False, disble_progress=True, print=True, dest=".", print_format=PrintFormat.YAML)
+        name="test", version=None, verbose=None, include="", exclude="", url="test_url", token="test_token", method=CloneMethod.SSH, naming=FolderNaming.NAME, archived=ArchivedResults.INCLUDE, file=None, concurrency=1, recursive=False, disble_progress=True, print=True, dest=".", print_format=PrintFormat.YAML, root_group=None)
     cli.parse_args = args_mock
 
     print_tree_mock = mock.Mock()
@@ -115,7 +115,7 @@ def test_missing_url(mock_tree):
 def test_empty_tree(mock_tree):
     args_mock = mock.Mock()
     args_mock.return_value = Node(
-        name="test", version=None, verbose=None, include="", exclude="", url="test_url", token="test_token", method=CloneMethod.SSH, naming=FolderNaming.NAME, archived=ArchivedResults.INCLUDE, file=None, concurrency=1, recursive=False, disble_progress=True, print=True, dest=".")
+        name="test", version=None, verbose=None, include="", exclude="", url="test_url", token="test_token", method=CloneMethod.SSH, naming=FolderNaming.NAME, archived=ArchivedResults.INCLUDE, file=None, concurrency=1, recursive=False, disble_progress=True, print=True, dest=".", root_group=None)
     cli.parse_args = args_mock
 
     with pytest.raises(SystemExit):

tests/test_gitlab_tree.py

@@ -142,5 +142,19 @@ def test_get_ca_path(monkeypatch):
     result = gitlab_tree.GitlabTree.get_ca_path()
     assert result == True
 
+def test_root_group(monkeypatch):
+    gl = gitlab_util.create_test_gitlab(monkeypatch, root_group='group')
+    gl.load_tree()
+    gitlab_util.validate_group(gl.root)
 
+def test_root_group_subgroup(monkeypatch):
+    gl = gitlab_util.create_test_gitlab(monkeypatch,
+                                        root_group='group/subgroup')
+    gl.load_tree()
+    gitlab_util.validate_subgroup(gl.root)
 
+def test_root_group_subgroup_name(monkeypatch):
+    gl = gitlab_util.create_test_gitlab(monkeypatch,
+                                        root_group='group / subgroup')
+    gl.load_tree()
+    gitlab_util.validate_subgroup(gl.root)