#107 Add possibility to filter shared projects

README.rst

@@ -115,6 +115,7 @@ Usage
     -x csv, --exclude csv
                             comma delimited list of glob patterns of paths to projects or groups to exclude from clone/pull
     -r, --recursive       clone/pull git submodules recursively
+    -S, --without_shared  exclude shared projects in the results
     --version             print the version
 
     examples:

gitlabber/cli.py

@@ -38,7 +38,7 @@ def main():
     excludes=split(args.exclude)
 
     tree = GitlabTree(args.url, args.token, args.method, args.naming, args.archived.api_value, includes,
-                      excludes, args.file, args.concurrency, args.recursive, args.verbose)
+                      excludes, args.file, args.concurrency, args.recursive, not args.without_shared, args.verbose)
     log.debug("Reading projects tree from gitlab at [%s]", args.url)
     tree.load_tree()
 
@@ -177,6 +177,12 @@ def parse_args(argv=None):
         action='store_true',
         default=False,
         help='clone/pull git submodules recursively')
+    parser.add_argument(
+        '-S',
+        '--without_shared',
+        action='store_true',
+        default=False,
+        help='exclude shared projects in the results')
     parser.add_argument(
         '--version',
         action='store_true',

gitlabber/gitlab_tree.py

@@ -16,7 +16,7 @@
 
 class GitlabTree:
 
-    def __init__(self, url, token, method, naming, archived=None, includes=[], excludes=[], in_file=None, concurrency=1, recursive=False, disable_progress=False):
+    def __init__(self, url, token, method, naming, archived=None, includes=[], excludes=[], in_file=None, concurrency=1, recursive=False, with_shared=True, disable_progress=False):
         self.includes = includes
         self.excludes = excludes
         self.url = url
@@ -29,6 +29,7 @@ def __init__(self, url, token, method, naming, archived=None, includes=[], exclu
         self.in_file = in_file
         self.concurrency = concurrency
         self.recursive = recursive
+        self.with_shared = with_shared
         self.disable_progress = disable_progress
         self.progress = ProgressBar('* loading tree', disable_progress)
 
@@ -101,7 +102,7 @@ def add_projects(self, parent, projects):
             self.progress.show_progress(node.name, 'project')
 
     def get_projects(self, group, parent):
-        projects = group.projects.list(as_list=False, archived=self.archived)
+        projects = group.projects.list(as_list=False, archived=self.archived, with_shared=self.with_shared)
         self.progress.update_progress_length(len(projects))
         self.add_projects(parent, projects)
 

tests/gitlab_test_utils.py

@@ -21,7 +21,7 @@
 
 
 class MockNode:
-    def __init__(self, id, name, url, subgroups=mock.MagicMock(), projects=mock.MagicMock(), parent_id=None):
+    def __init__(self, id, name, url, subgroups=mock.MagicMock(), projects=mock.MagicMock(), parent_id=None, archived=False, shared=False):
         self.id = id
         self.name = name
         self.path = name
@@ -32,27 +32,51 @@ def __init__(self, id, name, url, subgroups=mock.MagicMock(), projects=mock.Magi
         self.subgroups = subgroups
         self.projects = projects
         self.parent_id = parent_id
+        self.archived = archived
+        self.shared = shared
 
 
 class Listable:
-    def __init__(self, list_result, get_result=None, archive_result=None):
-        self.list_result = list_result
-        self.get_result = get_result
-        self.archive_result = archive_result
-
-    def list(self, as_list=False, archived=None):
-        if archived is None:
-            return [self.list_result, self.archive_result] if self.archive_result is not None else [self.list_result]
-        elif archived is True:
-            return [self.archive_result]
+    def __init__(self, *nodes: MockNode):
+        self.nodes = list(nodes)
+
+    def list(self, as_list=False, archived=None, with_shared=True):
+        filtered = filter(lambda it: self.is_visile(it, archived, with_shared), self.nodes)
+        return list(filtered)
+
+    def is_visile(self, node: MockNode, archived, with_shared):
+        if node.shared and with_shared is False:
+            return False
+        elif node.archived and archived is False:
+            return False
+        elif not node.archived and archived is True:
+            return False
         else:
-            return [self.list_result]
+            return True
+
+
+class Tree:
+    def __init__(self, roots: Listable):
+        self.all_nodes = []
+        for root_node in roots.nodes:
+            self.all_nodes.extend(self.get_all_nodes(root_node))
+        self.roots = roots
 
     def get(self, id):
-        if self.get_result is not None:
-            return self.get_result
-        else:
-            return self.list_result
+        return next(filter(lambda it: it.id == id, self.all_nodes))
+
+    def list(self, as_list=False, archived=None, with_shared=True):
+        return self.roots.list(as_list, archived, with_shared)
+
+    def get_all_nodes(self, node: MockNode):
+        nodes = [node]
+        if node.subgroups:
+            for sub_node in node.subgroups.nodes:
+                nodes.extend(self.get_all_nodes(sub_node))
+        if node.projects:
+            for sub_node in node.projects.nodes:
+                nodes.extend(self.get_all_nodes(sub_node))
+        return nodes
 
 
 def validate_root(root):
@@ -97,37 +121,56 @@ def create_test_gitlab(monkeypatch, includes=None, excludes=None, in_file=None):
     gl = gitlab_tree.GitlabTree(
         URL, TOKEN, "ssh", "name", includes=includes, excludes=excludes, in_file=in_file)
     projects = Listable(MockNode(2, PROJECT_NAME, PROJECT_URL))
-    subgroup_node = MockNode(2, SUBGROUP_NAME, SUBGROUP_URL, projects=projects)
-    subgroups = Listable(subgroup_node)
-    groups = Listable(MockNode(2, GROUP_NAME, GROUP_URL,
-                               subgroups=subgroups), subgroup_node)
-    monkeypatch.setattr(gl.gitlab, "groups", groups)
+    groups = Listable(
+        MockNode(2, GROUP_NAME, GROUP_URL, subgroups=Listable(
+            MockNode(3, SUBGROUP_NAME, SUBGROUP_URL, projects=projects)
+        ))
+    )
+    monkeypatch.setattr(gl.gitlab, "groups", Tree(groups))
     return gl
 
 
 def create_test_gitlab_with_toplevel_subgroups(monkeypatch):
     gl = gitlab_tree.GitlabTree(URL, TOKEN, "ssh", "path")
-    groups = Listable([MockNode(2, GROUP_NAME, GROUP_URL),
-                       MockNode(2, GROUP_NAME, GROUP_URL, parent_id=1)])
-    monkeypatch.setattr(gl.gitlab, "groups", groups)
+    groups = Listable(
+        MockNode(2, GROUP_NAME, GROUP_URL),
+        MockNode(3, GROUP_NAME, GROUP_URL, parent_id=1)
+    )
+    monkeypatch.setattr(gl.gitlab, "groups", Tree(groups))
     return gl
 
 
 def create_test_gitlab_with_archived(monkeypatch, includes=None, excludes=None, in_file=None, archived=None):
     gl = gitlab_tree.GitlabTree(
         URL, TOKEN, "ssh", "name", includes=includes, excludes=excludes, in_file=in_file, archived=archived)
-    project_node = MockNode(1, PROJECT_NAME, PROJECT_URL)
-    archived_project_node = MockNode(
-        2, "_archived_" + PROJECT_NAME, "_archived_" + PROJECT_URL)
-    projects = Listable(project_node, archive_result=archived_project_node)
-    subgroup_node = MockNode(2, SUBGROUP_NAME, SUBGROUP_URL, projects=projects)
-    archived_subgroup_node = MockNode(
-        2, "_archived_" + SUBGROUP_NAME, "_archived_" + SUBGROUP_URL, projects=projects)
-    subgroups = Listable(subgroup_node, archive_result=archived_subgroup_node)
-    archived_subgroups = Listable(archived_subgroup_node, archive_result=archived_subgroup_node)
-    group_node = MockNode(2, GROUP_NAME, GROUP_URL, subgroups=archived_subgroups)
-    archived_group_node = MockNode(2, "_archived_" + GROUP_NAME, "_archived_" + GROUP_URL, subgroups=archived_subgroups)
-    groups = Listable(group_node, get_result=subgroup_node, archive_result=archived_group_node)
-    monkeypatch.setattr(gl.gitlab, "groups", groups)
+    projects = Listable(
+        MockNode(11, PROJECT_NAME, PROJECT_URL),
+        MockNode(12, "_archived_" + PROJECT_NAME, "_archived_" + PROJECT_URL, archived=True)
+    )
+    monkeypatch.setattr(gl.gitlab, "groups", Tree(Listable(
+        MockNode(1, GROUP_NAME, GROUP_URL, subgroups=Listable(
+            MockNode(2, SUBGROUP_NAME, SUBGROUP_URL, projects=projects),
+            MockNode(3, SUBGROUP_NAME, SUBGROUP_URL, projects=projects, archived=True)
+        )),
+        MockNode(4, "_archived_" + GROUP_NAME, "_archived_" + GROUP_URL, archived=True, subgroups=Listable(
+            MockNode(5, SUBGROUP_NAME, SUBGROUP_URL, projects=projects),
+            MockNode(6, SUBGROUP_NAME, SUBGROUP_URL, projects=projects, archived=True)
+        ))
+    )))
     # gl.print_tree()
     return gl
+
+
+def create_test_gitlab_with_shared(monkeypatch, includes=None, excludes=None, in_file=None, with_shared=True):
+    gl = gitlab_tree.GitlabTree(
+        URL, TOKEN, "ssh", "name", includes=includes, excludes=excludes, in_file=in_file, with_shared=with_shared)
+
+    projects = Listable(
+        MockNode(11, PROJECT_NAME, PROJECT_URL),
+        MockNode(13, "_shared_" + PROJECT_NAME, "_shared_" + PROJECT_URL, shared=True)
+    )
+    monkeypatch.setattr(gl.gitlab, "groups", Tree(Listable(
+        MockNode(1, GROUP_NAME, GROUP_URL, projects=projects)
+    )))
+    return gl
+

tests/test_cli.py

@@ -36,7 +36,9 @@ def test_args_version():
 def test_args_logging(mock_tree, mock_log, mock_os, mock_sys, mock_logging):
     args_mock = mock.Mock()
     args_mock.return_value = Node(
-        name="test", version=None, verbose=True, include="", exclude="", url="test_url", token="test_token", method=CloneMethod.SSH, naming=FolderNaming.PATH, archived=ArchivedResults.INCLUDE, file=None, concurrency=1, recursive=False, disble_progress=True, print=None, dest=".")
+        name="test", version=None, verbose=True, include="", exclude="", url="test_url", token="test_token",
+        method=CloneMethod.SSH, naming=FolderNaming.PATH, archived=ArchivedResults.INCLUDE, file=None, concurrency=1,
+        recursive=False, disble_progress=True, print=None, dest=".", without_shared=False)
     cli.parse_args = args_mock
 
     mock_streamhandler = mock.Mock()
@@ -73,7 +75,10 @@ def test_args_include(mock_tree):
 def test_args_include(mock_tree):
     args_mock = mock.Mock()
     args_mock.return_value = Node(
-        name="test", version=None, verbose=None, include="", exclude="", url="test_url", token="test_token", method=CloneMethod.SSH, naming=FolderNaming.NAME, archived=ArchivedResults.INCLUDE, file=None, concurrency=1, recursive=False, disble_progress=True, print=True, dest=".", print_format=PrintFormat.YAML)
+        name="test", version=None, verbose=None, include="", exclude="", url="test_url", token="test_token",
+        method=CloneMethod.SSH, naming=FolderNaming.NAME, archived=ArchivedResults.INCLUDE, file=None, concurrency=1,
+        recursive=False, disble_progress=True, print=True, dest=".", print_format=PrintFormat.YAML,
+        without_shared=False)
     cli.parse_args = args_mock
 
     print_tree_mock = mock.Mock()
@@ -115,7 +120,9 @@ def test_missing_url(mock_tree):
 def test_empty_tree(mock_tree):
     args_mock = mock.Mock()
     args_mock.return_value = Node(
-        name="test", version=None, verbose=None, include="", exclude="", url="test_url", token="test_token", method=CloneMethod.SSH, naming=FolderNaming.NAME, archived=ArchivedResults.INCLUDE, file=None, concurrency=1, recursive=False, disble_progress=True, print=True, dest=".")
+        name="test", version=None, verbose=None, include="", exclude="", url="test_url", token="test_token",
+        method=CloneMethod.SSH, naming=FolderNaming.NAME, archived=ArchivedResults.INCLUDE, file=None, concurrency=1,
+        recursive=False, disble_progress=True, print=True, dest=".", without_shared=False)
     cli.parse_args = args_mock
 
     with pytest.raises(SystemExit):

tests/test_gitlab_tree.py

@@ -126,6 +126,31 @@ def test_archive_only(monkeypatch):
     assert "_archived_" in gl.root.children[0].name
     assert "_archived_" in gl.root.children[0].children[0].children[0].name
 
+
+def test_shared_included(monkeypatch):
+    gl = gitlab_util.create_test_gitlab_with_shared(monkeypatch, with_shared=True)
+
+    gl.load_tree()
+    gl.print_tree()
+    assert gl.root.is_leaf is False
+    assert len(gl.root.children) == 1
+    assert len(gl.root.children[0].children) == 2
+
+    assert "project" in gl.root.children[0].children[0].name
+    assert "_shared_" in gl.root.children[0].children[1].name
+
+
+def test_shared_excluded(monkeypatch):
+    gl = gitlab_util.create_test_gitlab_with_shared(monkeypatch, with_shared=False)
+
+    gl.load_tree()
+    gl.print_tree()
+    assert gl.root.is_leaf is False
+    assert len(gl.root.children) == 1
+    assert len(gl.root.children[0].children) == 1
+
+    assert "project" in gl.root.children[0].children[0].name
+
 def test_get_ca_path(monkeypatch):
     import os
     from gitlabber import gitlab_tree