From c1edd2c894d6f28903c70caa2ff5c7a6e892da26 Mon Sep 17 00:00:00 2001 From: James Edwards Date: Sun, 11 Feb 2024 10:49:54 -0700 Subject: [PATCH 1/8] all working --- README.md | 18 +++++---- git_fleximod/git_fleximod.py | 7 ++-- pyproject.toml | 6 +++ tests/conftest.py | 73 +++++++++++++++++++++++++---------- tests/test_checkout.py | 54 ++++++++++++++------------ tests/test_sparse_checkout.py | 52 ++++++++++++------------- 6 files changed, 127 insertions(+), 83 deletions(-) diff --git a/README.md b/README.md index 7c968e70b0..259ba8b54f 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ # git-fleximod -Flexible Submodule Management for Git +Flexible, Enhanced Submodule Management for Git ## Overview -Git-fleximod is a Python-based tool that extends Git's submodule capabilities, offering additional features for managing submodules in a more flexible and efficient way. +Git-fleximod is a Python-based tool that extends Git's submodule and sparse checkout capabilities, offering additional features for managing submodules in a more flexible and efficient way. ## Installation @@ -30,10 +30,10 @@ Git-fleximod is a Python-based tool that extends Git's submodule capabilities, o fxtag: Specify a specific tag or branch to checkout for a submodule. fxrequired: Mark a submodule's checkout behavior, with allowed values: - - T:T: Top-level and required (checked out only when this is the Toplevel module). - - T:F: Top-level and optional (checked out with --optional flag if this is the Toplevel module). - - I:T: Internal and required (always checked out). - - I:F: Internal and optional (checked out with --optional flag). + - ToplevelOnlyRequired: Top-level and required (checked out only when this is the Toplevel module). + - ToplevelOnlyOptional: Top-level and optional (checked out with --optional flag if this is the Toplevel module). + - AlwaysRequired: Always required (always checked out). + - AlwaysOptional: Always optional (checked out with --optional flag). fxsparse: Enable sparse checkout for a submodule, pointing to a file containing sparse checkout paths. fxurl: This is the url used in the test subcommand to assure that protected branches do not point to forks @@ -71,6 +71,7 @@ Example .gitmodules entry: path = src/physics/cosp2/src url = https://github.com/CFMIP/COSPv2.0 fxsparse = ../.cosp_sparse_checkout + fxrequired = AlwaysRequired fxtag = v2.1.4cesm ``` Explanation: @@ -80,14 +81,15 @@ should be checked out into the directory src/physics/cosp2/src relative to the .gitmodules directory. It should be checked out from the URL https://github.com/CFMIP/COSPv2.0 and use sparse checkout as described in the file ../.cosp_sparse_checkout relative to the path -directory. +directory. It should be checked out anytime this .gitmodules entry is +read. Additional example: ```ini, toml [submodule "cime"] path = cime url = https://github.com/jedwards4b/cime - fxrequired = T:T + fxrequired = ToplevelOnlyRequired fxtag = cime6.0.198_rme01 ``` diff --git a/git_fleximod/git_fleximod.py b/git_fleximod/git_fleximod.py index 1b6c357b94..df76127b38 100755 --- a/git_fleximod/git_fleximod.py +++ b/git_fleximod/git_fleximod.py @@ -24,9 +24,9 @@ def commandline_arguments(args=None): # explicitly listing a component overrides the optional flag if options.optional or options.components: - fxrequired = ["T:T", "T:F", "I:T"] + fxrequired = ["ToplevelOnlyRequired", "ToplevelOnlyOptional", "AlwaysRequired", "AlwaysOptional"] else: - fxrequired = ["T:T", "I:T"] + fxrequired = ["ToplevelOnlyRequired", "AlwaysRequired"] action = options.action if not action: @@ -66,6 +66,7 @@ def commandline_arguments(args=None): def submodule_sparse_checkout( root_dir, name, url, path, sparsefile, tag="master", fxhash=None ): + logger.info("Called sparse_checkout for {}".format(name)) # first create the module directory if not os.path.isdir(os.path.join(root_dir,path)): os.makedirs(os.path.join(root_dir,path)) @@ -344,7 +345,7 @@ def submodules_checkout(gitmodules, root_dir, requiredlist, force=False): url = gitmodules.get(name, "url") if fxrequired and fxrequired not in requiredlist: - if "T:F" == fxrequired: + if "Optional" in fxrequired: print("Skipping optional component {}".format(name)) continue diff --git a/pyproject.toml b/pyproject.toml index 6d0fa42157..772b154216 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,12 @@ pyfakefs = "^5.3.5" [tool.poetry.urls] "Bug Tracker" = "https://github.com/jedwards4b/git-fleximod/issues" +[tool.pytest.ini_options] +markers = [ + "skip_after_first: only run on first iteration" +] + [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" + diff --git a/tests/conftest.py b/tests/conftest.py index 1cb059dac6..d6fec44f3e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,39 +13,72 @@ def logger(): logger = logging.getLogger(__name__) return logger +@pytest.fixture(params=[ + {"subrepo_path": "modules/test", "submodule_name": "test_submodule", "gitmodules_content" : """ + [submodule "test_submodule"] + path = modules/test + url = https://github.com/ESMCI/mpi-serial.git + fxtag = MPIserial_2.4.0 + fxurl = https://github.com/ESMCI/mpi-serial.git + fxrequired = ToplevelOnlyRequired +"""}, + {"subrepo_path": "modules/test_optional", "submodule_name": "test_optional", "gitmodules_content": """ + [submodule "test_optional"] + path = modules/test_optional + url = https://github.com/ESMCI/mpi-serial.git + fxtag = MPIserial_2.4.0 + fxurl = https://github.com/ESMCI/mpi-serial.git + fxrequired = ToplevelOnlyRequired +"""}, + {"subrepo_path": "modules/test_alwaysoptional", "submodule_name": "test_alwaysoptional", "gitmodules_content": """ + [submodule "test_alwaysoptional"] + path = modules/test_alwaysoptional + url = https://github.com/ESMCI/mpi-serial.git + fxtag = MPIserial_2.3.0 + fxurl = https://github.com/ESMCI/mpi-serial.git + fxrequired = AlwaysOptional +"""}, + {"subrepo_path": "modules/test_sparse", "submodule_name": "test_sparse", "gitmodules_content": """ + [submodule "test_sparse"] + path = modules/test_sparse + url = https://github.com/ESMCI/mpi-serial.git + fxtag = MPIserial_2.5.0 + fxurl = https://github.com/ESMCI/mpi-serial.git + fxrequired = AlwaysRequired + fxsparse = ../.sparse_file_list +"""}, +]) + +def shared_repos(request): + return request.param + +@pytest.fixture +def test_repo(shared_repos, test_repo_base, logger): + subrepo_path = shared_repos["subrepo_path"] + submodule_name = shared_repos["submodule_name"] + + gitp = GitInterface(str(test_repo_base), logger) + gitp.git_operation("submodule", "add", "--depth","1","--name", submodule_name, "https://github.com/ESMCI/mpi-serial.git", subrepo_path) + assert test_repo_base.joinpath(".gitmodules").is_file() + return test_repo_base + @pytest.fixture def test_repo_base(tmp_path, logger): - test_dir = tmp_path / "test_repo" + test_dir = tmp_path / "testrepo" test_dir.mkdir() str_path = str(test_dir) gitp = GitInterface(str_path, logger) - #subprocess.run(["git", "init"], cwd=test_dir) assert test_dir.joinpath(".git").is_dir() + (test_dir / "modules").mkdir() return test_dir -@pytest.fixture(params=["modules/test"]) -def test_repo(request, test_repo_base, logger): - subrepo_path = request.param - gitp = GitInterface(str(test_repo_base), logger) - gitp.git_operation("submodule", "add", "--depth","1","--name","test_submodule", "https://github.com/ESMCI/mpi-serial.git", subrepo_path) - # subprocess.run( - assert test_repo_base.joinpath(".gitmodules").is_file() - return test_repo_base - @pytest.fixture -def git_fleximod(test_repo): +def git_fleximod(test_repo_base): def _run_fleximod(args, input=None): cmd = ["git", "fleximod"] + args.split() - result = subprocess.run(cmd, cwd=test_repo, input=input, + result = subprocess.run(cmd, cwd=test_repo_base, input=input, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) return result return _run_fleximod - -@pytest.fixture -def deinit_submodule(test_repo, logger): - def _deinit(submodule_path): - gitp = GitInterface(str(test_repo), logger) - gitp.git_operation( "submodule", "deinit", "-f", submodule_path) - yield _deinit diff --git a/tests/test_checkout.py b/tests/test_checkout.py index 791587628b..6d17ce18fd 100644 --- a/tests/test_checkout.py +++ b/tests/test_checkout.py @@ -1,32 +1,38 @@ import pytest from pathlib import Path -def test_basic_checkout(git_fleximod, test_repo): - # Prepare a simple .gitmodules - - gitmodules_content = """ - [submodule "test_submodule"] +@pytest.fixture(params=[ + {"subrepo_path": "modules/test", "submodule_name": "test_submodule", "gitmodules_content" : """ [submodule "test_submodule"] path = modules/test url = https://github.com/ESMCI/mpi-serial.git fxtag = MPIserial_2.4.0 fxurl = https://github.com/ESMCI/mpi-serial.git - fxrequired = T:T - """ - (test_repo / ".gitmodules").write_text(gitmodules_content) + fxrequired = ToplevelOnlyRequired +"""}, +]) +def test_config(request): + return request.param + +def test_basic_checkout(git_fleximod, test_repo, test_config): + # Prepare a simple .gitmodules + gm = test_config['gitmodules_content'] + file_path = (test_repo / ".gitmodules") + if not file_path.exists(): + file_path.write_text(gm) - # Run the command - result = git_fleximod("checkout") - - # Assertions - assert result.returncode == 0 - assert Path(test_repo / "modules/test").exists() # Did the submodule directory get created? - - status = git_fleximod("status") - - assert "test_submodule d82ce7c is out of sync with .gitmodules MPIserial_2.4.0" in status.stdout - - result = git_fleximod("update") - assert result.returncode == 0 - - status = git_fleximod("status") - assert "test_submodule at tag MPIserial_2.4.0" in status.stdout + # Run the command + result = git_fleximod("checkout test_submodule") + + # Assertions + assert result.returncode == 0 + assert Path(test_repo / "modules/test").exists() # Did the submodule directory get created? + + status = git_fleximod("status") + + assert "test_submodule d82ce7c is out of sync with .gitmodules MPIserial_2.4.0" in status.stdout + + result = git_fleximod("update") + assert result.returncode == 0 + + status = git_fleximod("status") + assert "test_submodule at tag MPIserial_2.4.0" in status.stdout diff --git a/tests/test_sparse_checkout.py b/tests/test_sparse_checkout.py index 0633802cb0..7276e18d35 100644 --- a/tests/test_sparse_checkout.py +++ b/tests/test_sparse_checkout.py @@ -3,36 +3,32 @@ from pathlib import Path from git_fleximod.gitinterface import GitInterface -def test_sparse_checkout(git_fleximod, test_repo_base): - # Prepare a simple .gitmodules - gitmodules_content = (test_repo_base / ".gitmodules").read_text() + """ - [submodule "test_sparse_submodule"] - path = modules/sparse_test - url = https://github.com/ESMCI/mpi-serial.git - fxtag = MPIserial_2.5.0 - fxurl = https://github.com/ESMCI/mpi-serial.git - fxsparse = ../.sparse_file_list - """ - (test_repo_base / ".gitmodules").write_text(gitmodules_content) +def test_sparse_checkout(shared_repos, git_fleximod, test_repo_base): + repo_name = shared_repos["submodule_name"] + if repo_name == "test_sparse": + gm = shared_repos["gitmodules_content"] + (test_repo_base / ".gitmodules").write_text(gm) - # Add the sparse checkout file - sparse_content = """m4 + # Add the sparse checkout file + sparse_content = """m4 """ - (test_repo_base / "modules" / ".sparse_file_list").write_text(sparse_content) - result = git_fleximod("checkout") - - # Assertions - assert result.returncode == 0 - assert Path(test_repo_base / "modules/sparse_test").exists() # Did the submodule directory get created? - assert Path(test_repo_base / "modules/sparse_test/m4").exists() # Did the submodule sparse directory get created? - assert not Path(test_repo_base / "modules/sparse_test/README").exists() # Did only the submodule sparse directory get created? - status = git_fleximod("status test_sparse_submodule") - - assert "test_sparse_submodule at tag MPIserial_2.5.0" in status.stdout + (test_repo_base / "modules" / ".sparse_file_list").write_text(sparse_content) - result = git_fleximod("update") - assert result.returncode == 0 + result = git_fleximod(f"checkout {repo_name}") + + # Assertions + assert result.returncode == 0 + assert Path(test_repo_base / f"modules/{repo_name}").exists() # Did the submodule directory get created? + assert Path(test_repo_base / f"modules/{repo_name}/m4").exists() # Did the submodule sparse directory get created? + assert not Path(test_repo_base / f"modules/{repo_name}/README").exists() # Did only the submodule sparse directory get created? + status = git_fleximod(f"status {repo_name}") + + assert f"{repo_name} at tag MPIserial_2.5.0" in status.stdout + + result = git_fleximod(f"update {repo_name}") + assert result.returncode == 0 + + status = git_fleximod(f"status {repo_name}") + assert f"{repo_name} at tag MPIserial_2.5.0" in status.stdout - status = git_fleximod("status test_sparse_submodule") - assert "test_sparse_submodule at tag MPIserial_2.5.0" in status.stdout From 03156218c7ddf4387fd5a8b74b8550ddae7d08da Mon Sep 17 00:00:00 2001 From: James Edwards Date: Sun, 11 Feb 2024 11:47:20 -0700 Subject: [PATCH 2/8] clean up and combine tests --- tests/conftest.py | 50 +++++++++++++++++++++++----------- tests/test_checkout.py | 51 +++++++++++++++-------------------- tests/test_sparse_checkout.py | 34 ----------------------- 3 files changed, 57 insertions(+), 78 deletions(-) delete mode 100644 tests/test_sparse_checkout.py diff --git a/tests/conftest.py b/tests/conftest.py index d6fec44f3e..ffff68956d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,7 +14,11 @@ def logger(): return logger @pytest.fixture(params=[ - {"subrepo_path": "modules/test", "submodule_name": "test_submodule", "gitmodules_content" : """ + {"subrepo_path": "modules/test", + "submodule_name": "test_submodule", + "status1" : "test_submodule d82ce7c is out of sync with .gitmodules MPIserial_2.4.0", + "status2" : "test_submodule at tag MPIserial_2.4.0", + "gitmodules_content" : """ [submodule "test_submodule"] path = modules/test url = https://github.com/ESMCI/mpi-serial.git @@ -22,7 +26,11 @@ def logger(): fxurl = https://github.com/ESMCI/mpi-serial.git fxrequired = ToplevelOnlyRequired """}, - {"subrepo_path": "modules/test_optional", "submodule_name": "test_optional", "gitmodules_content": """ + {"subrepo_path": "modules/test_optional", + "submodule_name": "test_optional", + "status1" : "test_optional d82ce7c is out of sync with .gitmodules MPIserial_2.4.0", + "status2" : "test_optional at tag MPIserial_2.4.0", + "gitmodules_content": """ [submodule "test_optional"] path = modules/test_optional url = https://github.com/ESMCI/mpi-serial.git @@ -30,7 +38,11 @@ def logger(): fxurl = https://github.com/ESMCI/mpi-serial.git fxrequired = ToplevelOnlyRequired """}, - {"subrepo_path": "modules/test_alwaysoptional", "submodule_name": "test_alwaysoptional", "gitmodules_content": """ + {"subrepo_path": "modules/test_alwaysoptional", + "submodule_name": "test_alwaysoptional", + "status1" : "test_alwaysoptional d82ce7c is out of sync with .gitmodules MPIserial_2.3.0", + "status2" : "test_alwaysoptional at tag MPIserial_2.3.0", + "gitmodules_content": """ [submodule "test_alwaysoptional"] path = modules/test_alwaysoptional url = https://github.com/ESMCI/mpi-serial.git @@ -38,7 +50,11 @@ def logger(): fxurl = https://github.com/ESMCI/mpi-serial.git fxrequired = AlwaysOptional """}, - {"subrepo_path": "modules/test_sparse", "submodule_name": "test_sparse", "gitmodules_content": """ + {"subrepo_path": "modules/test_sparse", + "submodule_name": "test_sparse", + "status1" : "test_sparse at tag MPIserial_2.5.0", + "status2" : "test_sparse at tag MPIserial_2.5.0", + "gitmodules_content": """ [submodule "test_sparse"] path = modules/test_sparse url = https://github.com/ESMCI/mpi-serial.git @@ -53,30 +69,34 @@ def shared_repos(request): return request.param @pytest.fixture -def test_repo(shared_repos, test_repo_base, logger): +def test_repo(shared_repos, tmp_path, logger): subrepo_path = shared_repos["subrepo_path"] submodule_name = shared_repos["submodule_name"] - - gitp = GitInterface(str(test_repo_base), logger) - gitp.git_operation("submodule", "add", "--depth","1","--name", submodule_name, "https://github.com/ESMCI/mpi-serial.git", subrepo_path) - assert test_repo_base.joinpath(".gitmodules").is_file() - return test_repo_base - -@pytest.fixture -def test_repo_base(tmp_path, logger): test_dir = tmp_path / "testrepo" test_dir.mkdir() str_path = str(test_dir) gitp = GitInterface(str_path, logger) assert test_dir.joinpath(".git").is_dir() (test_dir / "modules").mkdir() + if "sparse" in submodule_name: + (test_dir / subrepo_path).mkdir() + # Add the sparse checkout file + sparse_content = """m4 +""" + (test_dir / "modules" / ".sparse_file_list").write_text(sparse_content) + else: + gitp = GitInterface(str(test_dir), logger) + gitp.git_operation("submodule", "add", "--depth","1","--name", submodule_name, "https://github.com/ESMCI/mpi-serial.git", subrepo_path) + assert test_dir.joinpath(".gitmodules").is_file() + + return test_dir @pytest.fixture -def git_fleximod(test_repo_base): +def git_fleximod(test_repo): def _run_fleximod(args, input=None): cmd = ["git", "fleximod"] + args.split() - result = subprocess.run(cmd, cwd=test_repo_base, input=input, + result = subprocess.run(cmd, cwd=test_repo, input=input, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) return result diff --git a/tests/test_checkout.py b/tests/test_checkout.py index 6d17ce18fd..38bc5f9b28 100644 --- a/tests/test_checkout.py +++ b/tests/test_checkout.py @@ -1,38 +1,31 @@ import pytest from pathlib import Path - -@pytest.fixture(params=[ - {"subrepo_path": "modules/test", "submodule_name": "test_submodule", "gitmodules_content" : """ [submodule "test_submodule"] - path = modules/test - url = https://github.com/ESMCI/mpi-serial.git - fxtag = MPIserial_2.4.0 - fxurl = https://github.com/ESMCI/mpi-serial.git - fxrequired = ToplevelOnlyRequired -"""}, -]) -def test_config(request): - return request.param -def test_basic_checkout(git_fleximod, test_repo, test_config): +def test_basic_checkout(git_fleximod, test_repo, shared_repos): # Prepare a simple .gitmodules - gm = test_config['gitmodules_content'] + gm = shared_repos['gitmodules_content'] file_path = (test_repo / ".gitmodules") - if not file_path.exists(): - file_path.write_text(gm) + repo_name = shared_repos["submodule_name"] + repo_path = shared_repos["subrepo_path"] + + file_path.write_text(gm) - # Run the command - result = git_fleximod("checkout test_submodule") - - # Assertions - assert result.returncode == 0 - assert Path(test_repo / "modules/test").exists() # Did the submodule directory get created? - - status = git_fleximod("status") + # Run the command + result = git_fleximod(f"checkout {repo_name}") + + # Assertions + assert result.returncode == 0 + assert Path(test_repo / repo_path).exists() # Did the submodule directory get created? + if "sparse" in repo_name: + assert Path(test_repo / f"{repo_path}/m4").exists() # Did the submodule sparse directory get created? + assert not Path(test_repo / f"{repo_path}/README").exists() # Did only the submodule sparse directory get created? + + status = git_fleximod(f"status {repo_name}") - assert "test_submodule d82ce7c is out of sync with .gitmodules MPIserial_2.4.0" in status.stdout + assert shared_repos["status1"] in status.stdout - result = git_fleximod("update") - assert result.returncode == 0 + result = git_fleximod(f"update {repo_name}") + assert result.returncode == 0 - status = git_fleximod("status") - assert "test_submodule at tag MPIserial_2.4.0" in status.stdout + status = git_fleximod(f"status {repo_name}") + assert shared_repos["status2"] in status.stdout diff --git a/tests/test_sparse_checkout.py b/tests/test_sparse_checkout.py deleted file mode 100644 index 7276e18d35..0000000000 --- a/tests/test_sparse_checkout.py +++ /dev/null @@ -1,34 +0,0 @@ -import pytest -from shutil import rmtree -from pathlib import Path -from git_fleximod.gitinterface import GitInterface - -def test_sparse_checkout(shared_repos, git_fleximod, test_repo_base): - repo_name = shared_repos["submodule_name"] - if repo_name == "test_sparse": - gm = shared_repos["gitmodules_content"] - (test_repo_base / ".gitmodules").write_text(gm) - - # Add the sparse checkout file - sparse_content = """m4 -""" - - (test_repo_base / "modules" / ".sparse_file_list").write_text(sparse_content) - - result = git_fleximod(f"checkout {repo_name}") - - # Assertions - assert result.returncode == 0 - assert Path(test_repo_base / f"modules/{repo_name}").exists() # Did the submodule directory get created? - assert Path(test_repo_base / f"modules/{repo_name}/m4").exists() # Did the submodule sparse directory get created? - assert not Path(test_repo_base / f"modules/{repo_name}/README").exists() # Did only the submodule sparse directory get created? - status = git_fleximod(f"status {repo_name}") - - assert f"{repo_name} at tag MPIserial_2.5.0" in status.stdout - - result = git_fleximod(f"update {repo_name}") - assert result.returncode == 0 - - status = git_fleximod(f"status {repo_name}") - assert f"{repo_name} at tag MPIserial_2.5.0" in status.stdout - From 9602eb0b777c3e174027bc9ea121da907879bac6 Mon Sep 17 00:00:00 2001 From: James Edwards Date: Sun, 11 Feb 2024 14:43:38 -0700 Subject: [PATCH 3/8] all tests passing --- tests/conftest.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index ffff68956d..a6ba850893 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,8 +16,10 @@ def logger(): @pytest.fixture(params=[ {"subrepo_path": "modules/test", "submodule_name": "test_submodule", - "status1" : "test_submodule d82ce7c is out of sync with .gitmodules MPIserial_2.4.0", + "status1" : "test_submodule MPIserial_2.5.0-3-gd82ce7c is out of sync with .gitmodules MPIserial_2.4.0", "status2" : "test_submodule at tag MPIserial_2.4.0", + "status3" : "test_submodule MPIserial_2.5.0-3-gd82ce7c is out of sync with .gitmodules MPIserial_2.4.0", + "status4" : "test_submodule at tag MPIserial_2.4.0", "gitmodules_content" : """ [submodule "test_submodule"] path = modules/test @@ -28,20 +30,24 @@ def logger(): """}, {"subrepo_path": "modules/test_optional", "submodule_name": "test_optional", - "status1" : "test_optional d82ce7c is out of sync with .gitmodules MPIserial_2.4.0", + "status1" : "test_optional MPIserial_2.5.0-3-gd82ce7c is out of sync with .gitmodules MPIserial_2.4.0", "status2" : "test_optional at tag MPIserial_2.4.0", + "status3" : "test_optional not checked out, aligned at tag MPIserial_2.4.0", + "status4" : "test_optional at tag MPIserial_2.4.0", "gitmodules_content": """ [submodule "test_optional"] path = modules/test_optional url = https://github.com/ESMCI/mpi-serial.git fxtag = MPIserial_2.4.0 fxurl = https://github.com/ESMCI/mpi-serial.git - fxrequired = ToplevelOnlyRequired + fxrequired = ToplevelOnlyOptional """}, {"subrepo_path": "modules/test_alwaysoptional", "submodule_name": "test_alwaysoptional", - "status1" : "test_alwaysoptional d82ce7c is out of sync with .gitmodules MPIserial_2.3.0", + "status1" : "test_alwaysoptional MPIserial_2.5.0-3-gd82ce7c is out of sync with .gitmodules MPIserial_2.3.0", "status2" : "test_alwaysoptional at tag MPIserial_2.3.0", + "status3" : "test_alwaysoptional not checked out, aligned at tag MPIserial_2.3.0", + "status4" : "test_alwaysoptional at tag MPIserial_2.3.0", "gitmodules_content": """ [submodule "test_alwaysoptional"] path = modules/test_alwaysoptional @@ -54,6 +60,8 @@ def logger(): "submodule_name": "test_sparse", "status1" : "test_sparse at tag MPIserial_2.5.0", "status2" : "test_sparse at tag MPIserial_2.5.0", + "status3" : "test_sparse at tag MPIserial_2.5.0", + "status4" : "test_sparse at tag MPIserial_2.5.0", "gitmodules_content": """ [submodule "test_sparse"] path = modules/test_sparse @@ -84,13 +92,18 @@ def test_repo(shared_repos, tmp_path, logger): sparse_content = """m4 """ (test_dir / "modules" / ".sparse_file_list").write_text(sparse_content) + gitp.git_operation("add","modules/.sparse_file_list") else: gitp = GitInterface(str(test_dir), logger) gitp.git_operation("submodule", "add", "--depth","1","--name", submodule_name, "https://github.com/ESMCI/mpi-serial.git", subrepo_path) assert test_dir.joinpath(".gitmodules").is_file() + gitp.git_operation("add",subrepo_path) + gitp.git_operation("commit","-a","-m","\"add submod\"") + test_dir2 = tmp_path / "testrepo2" + gitp.git_operation("clone",test_dir,test_dir2) + return test_dir2 - - return test_dir + @pytest.fixture def git_fleximod(test_repo): From b5c4515d0e9d1a48de659f96da99e6a8f3c20ed5 Mon Sep 17 00:00:00 2001 From: James Edwards Date: Sun, 11 Feb 2024 15:52:38 -0700 Subject: [PATCH 4/8] thought it was working?? --- tests/test_required.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 tests/test_required.py diff --git a/tests/test_required.py b/tests/test_required.py new file mode 100644 index 0000000000..0997db69f7 --- /dev/null +++ b/tests/test_required.py @@ -0,0 +1,33 @@ +import pytest +from pathlib import Path + +def test_required(git_fleximod, test_repo, shared_repos): + file_path = (test_repo / ".gitmodules") + gm = shared_repos["gitmodules_content"] + repo_name = shared_repos["submodule_name"] + + status = git_fleximod(f"status {repo_name}") + print(f"stdout is {status.stdout}") + if file_path.exists(): + with file_path.open("r") as f: + gitmodules_content = f.read() + # add the entry if it does not exist + if repo_name not in gitmodules_content: + file_path.write_text(gitmodules_content+gm) + # or if it is incomplete + elif gm not in gitmodules_content: + file_path.write_text(gm) + else: + file_path.write_text(gm) + status = git_fleximod(f"status {repo_name}") + result = git_fleximod("checkout") + assert result.returncode == 0 + status = git_fleximod(f"status {repo_name}") + assert shared_repos["status3"] in status.stdout + if "not checked out" in status.stdout: + status = git_fleximod(f"checkout {repo_name}") + assert result.returncode == 0 + status = git_fleximod(f"update {repo_name}") + assert result.returncode == 0 + status = git_fleximod(f"status {repo_name}") + assert shared_repos["status4"] in status.stdout From a3ce6116065af2e8e86bd3d7d6de06bc215a6e58 Mon Sep 17 00:00:00 2001 From: James Edwards Date: Tue, 13 Feb 2024 15:50:55 -0700 Subject: [PATCH 5/8] test_complex is still a work in progress --- tests/conftest.py | 18 +++++++++++++----- tests/test_checkout.py | 2 +- tests/test_complex.py | 35 +++++++++++++++++++++++++++++++++++ tests/test_required.py | 11 ++++------- 4 files changed, 53 insertions(+), 13 deletions(-) create mode 100644 tests/test_complex.py diff --git a/tests/conftest.py b/tests/conftest.py index a6ba850893..9bd6a758b2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,7 +13,7 @@ def logger(): logger = logging.getLogger(__name__) return logger -@pytest.fixture(params=[ +all_repos=[ {"subrepo_path": "modules/test", "submodule_name": "test_submodule", "status1" : "test_submodule MPIserial_2.5.0-3-gd82ce7c is out of sync with .gitmodules MPIserial_2.4.0", @@ -71,11 +71,21 @@ def logger(): fxrequired = AlwaysRequired fxsparse = ../.sparse_file_list """}, -]) +] +@pytest.fixture(params=all_repos) def shared_repos(request): return request.param +@pytest.fixture +def get_all_repos(): + return all_repos + +def write_sparse_checkout_file(fp): + sparse_content = """m4 +""" + fp.write_text(sparse_content) + @pytest.fixture def test_repo(shared_repos, tmp_path, logger): subrepo_path = shared_repos["subrepo_path"] @@ -89,9 +99,7 @@ def test_repo(shared_repos, tmp_path, logger): if "sparse" in submodule_name: (test_dir / subrepo_path).mkdir() # Add the sparse checkout file - sparse_content = """m4 -""" - (test_dir / "modules" / ".sparse_file_list").write_text(sparse_content) + write_sparse_checkout_file(test_dir / "modules" / ".sparse_file_list") gitp.git_operation("add","modules/.sparse_file_list") else: gitp = GitInterface(str(test_dir), logger) diff --git a/tests/test_checkout.py b/tests/test_checkout.py index 38bc5f9b28..fd8ffc1996 100644 --- a/tests/test_checkout.py +++ b/tests/test_checkout.py @@ -14,7 +14,7 @@ def test_basic_checkout(git_fleximod, test_repo, shared_repos): result = git_fleximod(f"checkout {repo_name}") # Assertions - assert result.returncode == 0 + assert result.returncode == 0 assert Path(test_repo / repo_path).exists() # Did the submodule directory get created? if "sparse" in repo_name: assert Path(test_repo / f"{repo_path}/m4").exists() # Did the submodule sparse directory get created? diff --git a/tests/test_complex.py b/tests/test_complex.py new file mode 100644 index 0000000000..628a92d837 --- /dev/null +++ b/tests/test_complex.py @@ -0,0 +1,35 @@ +import pytest +from pathlib import Path +from git_fleximod.gitinterface import GitInterface + +def test_complex_checkout(git_fleximod, get_all_repos, test_repo, request, logger): + gitp = None + for repo in get_all_repos: + repo_name = repo["submodule_name"] + gm = repo["gitmodules_content"] + if "shared_repos0" in request.node.name: + if not gitp: + gitp = GitInterface(str(test_repo), logger) + file_path = (test_repo / ".gitmodules") + if file_path.exists(): + with file_path.open("r") as f: + gitmodules_content = f.read() + print(f"content={gitmodules_content}") + print(f"gm={gm}") + # add the entry if it does not exist + if repo_name not in gitmodules_content: + file_path.write_text(gitmodules_content+gm) + # or if it is incomplete + elif gm not in gitmodules_content: + file_path.write_text(gm) + + else: + file_path.write_text(gm) + if "sparse" in repo_name: + print(f"writing sparse_file_list in {test_repo}") + write_sparse_checkout_file(test_repo / "modules" / ".sparse_file_list") + gitp.git_operation("add","modules/.sparse_file_list") + gitp.git_operation("commit","-a","-m","\"add submod\"") + + + assert(False) diff --git a/tests/test_required.py b/tests/test_required.py index 0997db69f7..b4d99c5642 100644 --- a/tests/test_required.py +++ b/tests/test_required.py @@ -5,17 +5,14 @@ def test_required(git_fleximod, test_repo, shared_repos): file_path = (test_repo / ".gitmodules") gm = shared_repos["gitmodules_content"] repo_name = shared_repos["submodule_name"] - - status = git_fleximod(f"status {repo_name}") - print(f"stdout is {status.stdout}") if file_path.exists(): with file_path.open("r") as f: gitmodules_content = f.read() # add the entry if it does not exist - if repo_name not in gitmodules_content: - file_path.write_text(gitmodules_content+gm) - # or if it is incomplete - elif gm not in gitmodules_content: + if repo_name not in gitmodules_content: + file_path.write_text(gitmodules_content+gm) + # or if it is incomplete + elif gm not in gitmodules_content: file_path.write_text(gm) else: file_path.write_text(gm) From 7c47c05ffbd4d329e94ada655c9a467e8f968585 Mon Sep 17 00:00:00 2001 From: James Edwards Date: Wed, 14 Feb 2024 09:18:43 -0700 Subject: [PATCH 6/8] restructure of tests working --- tests/conftest.py | 20 +++++++++++++++----- tests/test_b_update.py | 4 ++-- tests/test_c_required.py | 12 ++++-------- tests/test_d_complex.py | 34 +++------------------------------- 4 files changed, 24 insertions(+), 46 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 9bd6a758b2..fd3678f9ee 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,7 +18,7 @@ def logger(): "submodule_name": "test_submodule", "status1" : "test_submodule MPIserial_2.5.0-3-gd82ce7c is out of sync with .gitmodules MPIserial_2.4.0", "status2" : "test_submodule at tag MPIserial_2.4.0", - "status3" : "test_submodule MPIserial_2.5.0-3-gd82ce7c is out of sync with .gitmodules MPIserial_2.4.0", + "status3" : "test_submodule at tag MPIserial_2.4.0", "status4" : "test_submodule at tag MPIserial_2.4.0", "gitmodules_content" : """ [submodule "test_submodule"] @@ -84,6 +84,7 @@ def get_all_repos(): def write_sparse_checkout_file(fp): sparse_content = """m4 """ + print(f"writing sparse_file_list \n") fp.write_text(sparse_content) @pytest.fixture @@ -110,14 +111,23 @@ def test_repo(shared_repos, tmp_path, logger): test_dir2 = tmp_path / "testrepo2" gitp.git_operation("clone",test_dir,test_dir2) return test_dir2 - + +@pytest.fixture +def complex_repo(tmp_path, logger): + test_dir = tmp_path / "testcomplex" + test_dir.mkdir() + str_path = str(test_dir) + gitp = GitInterface(str_path, logger) + gitp.git_operation("remote", "add", "origin", "https://github.com/jedwards4b/fleximod-test") + gitp.git_operation("fetch", "origin", "main") + return test_dir @pytest.fixture -def git_fleximod(test_repo): - def _run_fleximod(args, input=None): +def git_fleximod(): + def _run_fleximod(path, args, input=None): cmd = ["git", "fleximod"] + args.split() - result = subprocess.run(cmd, cwd=test_repo, input=input, + result = subprocess.run(cmd, cwd=path, input=input, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) return result diff --git a/tests/test_b_update.py b/tests/test_b_update.py index e03daa3974..159f1cfae0 100644 --- a/tests/test_b_update.py +++ b/tests/test_b_update.py @@ -11,7 +11,7 @@ def test_basic_checkout(git_fleximod, test_repo, shared_repos): file_path.write_text(gm) # Run the command - result = git_fleximod(f"update {repo_name}") + result = git_fleximod(test_repo, f"update {repo_name}") # Assertions assert result.returncode == 0 @@ -20,7 +20,7 @@ def test_basic_checkout(git_fleximod, test_repo, shared_repos): assert Path(test_repo / f"{repo_path}/m4").exists() # Did the submodule sparse directory get created? assert not Path(test_repo / f"{repo_path}/README").exists() # Did only the submodule sparse directory get created? - status = git_fleximod(f"status {repo_name}") + status = git_fleximod(test_repo, f"status {repo_name}") assert shared_repos["status2"] in status.stdout diff --git a/tests/test_c_required.py b/tests/test_c_required.py index b4d99c5642..d6cc040b8d 100644 --- a/tests/test_c_required.py +++ b/tests/test_c_required.py @@ -16,15 +16,11 @@ def test_required(git_fleximod, test_repo, shared_repos): file_path.write_text(gm) else: file_path.write_text(gm) - status = git_fleximod(f"status {repo_name}") - result = git_fleximod("checkout") + result = git_fleximod(test_repo, "update") assert result.returncode == 0 - status = git_fleximod(f"status {repo_name}") + status = git_fleximod(test_repo, f"status {repo_name}") assert shared_repos["status3"] in status.stdout - if "not checked out" in status.stdout: - status = git_fleximod(f"checkout {repo_name}") - assert result.returncode == 0 - status = git_fleximod(f"update {repo_name}") + status = git_fleximod(test_repo, f"update {repo_name}") assert result.returncode == 0 - status = git_fleximod(f"status {repo_name}") + status = git_fleximod(test_repo, f"status {repo_name}") assert shared_repos["status4"] in status.stdout diff --git a/tests/test_d_complex.py b/tests/test_d_complex.py index 628a92d837..2983b0193f 100644 --- a/tests/test_d_complex.py +++ b/tests/test_d_complex.py @@ -2,34 +2,6 @@ from pathlib import Path from git_fleximod.gitinterface import GitInterface -def test_complex_checkout(git_fleximod, get_all_repos, test_repo, request, logger): - gitp = None - for repo in get_all_repos: - repo_name = repo["submodule_name"] - gm = repo["gitmodules_content"] - if "shared_repos0" in request.node.name: - if not gitp: - gitp = GitInterface(str(test_repo), logger) - file_path = (test_repo / ".gitmodules") - if file_path.exists(): - with file_path.open("r") as f: - gitmodules_content = f.read() - print(f"content={gitmodules_content}") - print(f"gm={gm}") - # add the entry if it does not exist - if repo_name not in gitmodules_content: - file_path.write_text(gitmodules_content+gm) - # or if it is incomplete - elif gm not in gitmodules_content: - file_path.write_text(gm) - - else: - file_path.write_text(gm) - if "sparse" in repo_name: - print(f"writing sparse_file_list in {test_repo}") - write_sparse_checkout_file(test_repo / "modules" / ".sparse_file_list") - gitp.git_operation("add","modules/.sparse_file_list") - gitp.git_operation("commit","-a","-m","\"add submod\"") - - - assert(False) +def test_complex_checkout(git_fleximod, complex_repo, logger): + status = git_fleximod(complex_repo, "status") + print(status) From e6d4838ebeee5a7ce408aa098d9cfab0cd34d405 Mon Sep 17 00:00:00 2001 From: James Edwards Date: Fri, 16 Feb 2024 17:40:26 -0700 Subject: [PATCH 7/8] all tests are passing and complete --- README.md | 6 +- git_fleximod/git_fleximod.py | 167 +++++++++++++++++++---------------- tests/conftest.py | 13 +-- tests/test_c_required.py | 4 + tests/test_d_complex.py | 62 ++++++++++++- 5 files changed, 167 insertions(+), 85 deletions(-) diff --git a/README.md b/README.md index 259ba8b54f..94d85c4150 100644 --- a/README.md +++ b/README.md @@ -30,8 +30,8 @@ Git-fleximod is a Python-based tool that extends Git's submodule and sparse chec fxtag: Specify a specific tag or branch to checkout for a submodule. fxrequired: Mark a submodule's checkout behavior, with allowed values: - - ToplevelOnlyRequired: Top-level and required (checked out only when this is the Toplevel module). - - ToplevelOnlyOptional: Top-level and optional (checked out with --optional flag if this is the Toplevel module). + - ToplevelRequired: Top-level and required (checked out only when this is the Toplevel module). + - ToplevelOptional: Top-level and optional (checked out with --optional flag if this is the Toplevel module). - AlwaysRequired: Always required (always checked out). - AlwaysOptional: Always optional (checked out with --optional flag). fxsparse: Enable sparse checkout for a submodule, pointing to a file containing sparse checkout paths. @@ -89,7 +89,7 @@ Additional example: [submodule "cime"] path = cime url = https://github.com/jedwards4b/cime - fxrequired = ToplevelOnlyRequired + fxrequired = ToplevelRequired fxtag = cime6.0.198_rme01 ``` diff --git a/git_fleximod/git_fleximod.py b/git_fleximod/git_fleximod.py index f81e713814..08f31fa778 100755 --- a/git_fleximod/git_fleximod.py +++ b/git_fleximod/git_fleximod.py @@ -12,7 +12,8 @@ # logger variable is global logger = None - +def fxrequired_allowed_values(): + return ['ToplevelRequired', 'ToplevelOptional', 'AlwaysRequired', 'AlwaysOptional'] def commandline_arguments(args=None): parser = cli.get_parser() @@ -24,9 +25,9 @@ def commandline_arguments(args=None): # explicitly listing a component overrides the optional flag if options.optional or options.components: - fxrequired = ["ToplevelOnlyRequired", "ToplevelOnlyOptional", "AlwaysRequired", "AlwaysOptional"] + fxrequired = ["ToplevelRequired", "ToplevelOptional", "AlwaysRequired", "AlwaysOptional"] else: - fxrequired = ["ToplevelOnlyRequired", "AlwaysRequired"] + fxrequired = ["ToplevelRequired", "AlwaysRequired"] action = options.action if not action: @@ -67,12 +68,16 @@ def submodule_sparse_checkout( root_dir, name, url, path, sparsefile, tag="master", fxhash=None ): logger.info("Called sparse_checkout for {}".format(name)) + rgit = GitInterface(root_dir, logger) + superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree") + if superroot: + gitroot = superroot + else: + gitroot = root_dir + assert(os.path.isdir(os.path.join(gitroot,".git"))) # first create the module directory if not os.path.isdir(os.path.join(root_dir,path)): os.makedirs(os.path.join(root_dir,path)) - # Check first if the module is already defined - # and the sparse-checkout file exists - git = GitInterface(root_dir, logger) # initialize a new git repo and set the sparse checkout flag sprep_repo = os.path.join(root_dir, path) @@ -92,36 +97,39 @@ def submodule_sparse_checkout( sprepo_git.config_set_value("core", "sparseCheckout", "true") # set the repository remote - sprepo_git.git_operation("remote", "add", "origin", url) - superroot = git.git_operation("rev-parse", "--show-superproject-working-tree") - if os.path.isfile(os.path.join(root_dir, ".git")): + logger.info("Setting remote origin in {}/{}".format(root_dir,path)) + status = sprepo_git.git_operation("remote", "-v") + if url not in status: + sprepo_git.git_operation("remote", "add", "origin", url) + + topgit = os.path.join(gitroot,".git") + + if gitroot != root_dir and os.path.isfile(os.path.join(root_dir, ".git")): with open(os.path.join(root_dir, ".git")) as f: - gitpath = os.path.abspath(os.path.join(root_dir, f.read().split()[1])) - topgit = os.path.abspath(os.path.join(gitpath, "modules")) + gitpath = os.path.relpath(os.path.join(root_dir, f.read().split()[1]), start=os.path.join(root_dir,path)) + topgit = os.path.join(gitpath, "modules") else: - topgit = os.path.abspath(os.path.join(root_dir, ".git", "modules")) - - if not os.path.isdir(topgit): - os.makedirs(topgit) - topgit = os.path.join(topgit, name) - logger.debug( - "root_dir is {} topgit is {} superroot is {}".format( - root_dir, topgit, superroot - ) - ) - + topgit = os.path.relpath(os.path.join(root_dir, ".git", "modules"), start=os.path.join(root_dir,path)) + + with utils.pushd(sprep_repo): + if not os.path.isdir(topgit): + os.makedirs(topgit) + topgit += os.sep + name + if os.path.isdir(os.path.join(root_dir, path, ".git")): - shutil.move(os.path.join(root_dir, path, ".git"), topgit) - with open(os.path.join(root_dir, path, ".git"), "w") as f: - f.write("gitdir: " + os.path.relpath(topgit, os.path.join(root_dir, path))) - - gitsparse = os.path.abspath(os.path.join(topgit, "info", "sparse-checkout")) - if os.path.isfile(gitsparse): - logger.warning("submodule {} is already initialized".format(name)) - return - - shutil.copy(os.path.join(root_dir, path, sparsefile), gitsparse) + with utils.pushd(sprep_repo): + shutil.move(".git", topgit) + with open(".git", "w") as f: + f.write("gitdir: " + os.path.relpath(topgit)) + # assert(os.path.isdir(os.path.relpath(topgit, start=sprep_repo))) + gitsparse = os.path.abspath(os.path.join(topgit, "info", "sparse-checkout")) + if os.path.isfile(gitsparse): + logger.warning("submodule {} is already initialized {}".format(name, topgit)) + return + + with utils.pushd(sprep_repo): + shutil.copy(sparsefile, gitsparse) # Finally checkout the repo if fxhash: @@ -132,28 +140,31 @@ def submodule_sparse_checkout( sprepo_git.git_operation("fetch", "--depth=1", "origin", "--tags") sprepo_git.git_operation("checkout", tag) print(f"Successfully checked out {name:>20} at {tag}") - + rgit.config_set_value(f'submodule "{name}"',"active","true") + rgit.config_set_value(f'submodule "{name}"',"url",url) def single_submodule_checkout( - root, name, path, url=None, tag=None, force=False, fxhash=None + root, name, path, url=None, tag=None, force=False, fxhash=None, optional=False ): git = GitInterface(root, logger) repodir = os.path.join(root, path) - if os.path.exists(os.path.join(repodir, ".git")): - logger.info("Submodule {} already checked out".format(name)) - return + logger.info("Checkout {} into {}/{}".format(name,root,path)) # if url is provided update to the new url tmpurl = None + repo_exists = False + if os.path.exists(os.path.join(repodir, ".git")): + logger.info("Submodule {} already checked out".format(name)) + repo_exists = True # Look for a .gitmodules file in the newly checkedout repo - if url: + if not repo_exists and url: # ssh urls cause problems for those who dont have git accounts with ssh keys defined # but cime has one since e3sm prefers ssh to https, because the .gitmodules file was # opened with a GitModules object we don't need to worry about restoring the file here # it will be done by the GitModules class if url.startswith("git@"): tmpurl = url - url = url.replace("git@github.com:", "https://github.com") + url = url.replace("git@github.com:", "https://github.com/") git.git_operation("clone", url, path) smgit = GitInterface(repodir, logger) if not tag and not fxhash: @@ -170,26 +181,26 @@ def single_submodule_checkout( if line.startswith("gitdir: "): rootdotgit = line[8:].rstrip() - newpath = os.path.abspath(os.path.join(root, rootdotgit, "modules", path)) - if not os.path.isdir(os.path.join(newpath, os.pardir)): - os.makedirs(os.path.abspath(os.path.join(newpath, os.pardir))) - + newpath = os.path.abspath(os.path.join(root, rootdotgit, "modules", name)) shutil.move(os.path.join(repodir, ".git"), newpath) with open(os.path.join(repodir, ".git"), "w") as f: - f.write("gitdir: " + newpath) + f.write("gitdir: " + os.path.relpath(newpath, start=repodir)) - if not tmpurl: + if not repo_exists or not tmpurl: logger.debug(git.git_operation("submodule", "update", "--init", "--", path)) if os.path.exists(os.path.join(repodir, ".gitmodules")): # recursively handle this checkout print(f"Recursively checking out submodules of {name} {repodir} {url}") gitmodules = GitModules(logger, confpath=repodir) - submodules_checkout(gitmodules, repodir, ["I:T"], force=force) + requiredlist = ["AlwaysRequired"] + if optional: + requiredlist.append("AlwaysOptional") + submodules_checkout(gitmodules, repodir, requiredlist, force=force) if os.path.exists(os.path.join(repodir, ".git")): - print(f"Successfully checked out {name}") + print(f"Successfully checked out {name} {repodir}") else: - utils.fatal_error(f"Failed to checkout {name}") + utils.fatal_error(f"Failed to checkout {name} {repo_exists} {tmpurl} {repodir} {path}") if tmpurl: print(git.git_operation("restore", ".gitmodules")) @@ -289,33 +300,37 @@ def submodules_update(gitmodules, root_dir, requiredlist, force): fxhash = gitmodules.get(name, "fxhash") path = gitmodules.get(name, "path") url = gitmodules.get(name, "url") - logger.info("name={} path={} url={} fxtag={}".format(name, path, url, fxtag)) - if not os.path.exists(os.path.join(path, ".git")): - fxrequired = gitmodules.get(name, "fxrequired") - fxsparse = gitmodules.get(name, "fxsparse") - - if fxrequired and fxrequired not in requiredlist: - if "T:F" == fxrequired: - print("Skipping optional component {}".format(name)) - continue - - if fxsparse: - logger.debug( - "Callng submodule_sparse_checkout({}, {}, {}, {}, {}, {}".format( - root_dir, name, url, path, fxsparse, fxtag - ) - ) - submodule_sparse_checkout( - root_dir, name, url, path, fxsparse, tag=fxtag, fxhash=fxhash - ) - else: - logger.debug( - "Calling submodule_checkout({},{},{})".format(root_dir, name, path) + logger.info("name={} path={} url={} fxtag={} requiredlist={}".format(name,os.path.join(root_dir, path), url, fxtag, requiredlist)) +# if not os.path.exists(os.path.join(root_dir,path, ".git")): + fxrequired = gitmodules.get(name, "fxrequired") + assert(fxrequired in fxrequired_allowed_values()) + rgit = GitInterface(root_dir, logger) + superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree") + + fxsparse = gitmodules.get(name, "fxsparse") + + if fxrequired and (superroot and "Toplevel" in fxrequired) or fxrequired not in requiredlist: + if "ToplevelOptional" == fxrequired: + print("Skipping optional component {}".format(name)) + continue + if fxsparse: + logger.debug( + "Callng submodule_sparse_checkout({}, {}, {}, {}, {}, {}".format( + root_dir, name, url, path, fxsparse, fxtag ) + ) + submodule_sparse_checkout( + root_dir, name, url, path, fxsparse, tag=fxtag, fxhash=fxhash + ) + else: + logger.info( + "Calling submodule_checkout({},{},{},{})".format(root_dir, name, path,url) + ) - single_submodule_checkout( - root_dir, name, path, url=url, tag=fxtag, force=force, fxhash=fxhash - ) + single_submodule_checkout( + root_dir, name, path, url=url, tag=fxtag, force=force, + fxhash=fxhash, optional=("AlwaysOptional" in requiredlist) + ) if os.path.exists(os.path.join(path, ".git")): @@ -371,7 +386,6 @@ def submodules_checkout(gitmodules, root_dir, requiredlist, force=False): fxhash = gitmodules.get(name, "fxhash") path = gitmodules.get(name, "path") url = gitmodules.get(name, "url") - if fxrequired and fxrequired not in requiredlist: if "Optional" in fxrequired: print("Skipping optional component {}".format(name)) @@ -392,7 +406,8 @@ def submodules_checkout(gitmodules, root_dir, requiredlist, force=False): ) single_submodule_checkout( - root_dir, name, path, url=url, tag=fxtag, force=force, fxhash=fxhash + root_dir, name, path, url=url, tag=fxtag, force=force, fxhash=fxhash, + optional = "AlwaysOptional" in requiredlist ) @@ -441,7 +456,7 @@ def main(): ) root_dir = os.path.dirname(file_path) - logger.info("root_dir is {}".format(root_dir)) + logger.info("root_dir is {} includelist={} excludelist={}".format(root_dir, includelist, excludelist)) gitmodules = GitModules( logger, confpath=root_dir, diff --git a/tests/conftest.py b/tests/conftest.py index fd3678f9ee..b88fa61d27 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,7 +26,7 @@ def logger(): url = https://github.com/ESMCI/mpi-serial.git fxtag = MPIserial_2.4.0 fxurl = https://github.com/ESMCI/mpi-serial.git - fxrequired = ToplevelOnlyRequired + fxrequired = ToplevelRequired """}, {"subrepo_path": "modules/test_optional", "submodule_name": "test_optional", @@ -35,12 +35,12 @@ def logger(): "status3" : "test_optional not checked out, aligned at tag MPIserial_2.4.0", "status4" : "test_optional at tag MPIserial_2.4.0", "gitmodules_content": """ - [submodule "test_optional"] + [submodule "test_optional"] path = modules/test_optional url = https://github.com/ESMCI/mpi-serial.git fxtag = MPIserial_2.4.0 fxurl = https://github.com/ESMCI/mpi-serial.git - fxrequired = ToplevelOnlyOptional + fxrequired = ToplevelOptional """}, {"subrepo_path": "modules/test_alwaysoptional", "submodule_name": "test_alwaysoptional", @@ -84,7 +84,6 @@ def get_all_repos(): def write_sparse_checkout_file(fp): sparse_content = """m4 """ - print(f"writing sparse_file_list \n") fp.write_text(sparse_content) @pytest.fixture @@ -119,8 +118,9 @@ def complex_repo(tmp_path, logger): test_dir.mkdir() str_path = str(test_dir) gitp = GitInterface(str_path, logger) - gitp.git_operation("remote", "add", "origin", "https://github.com/jedwards4b/fleximod-test") + gitp.git_operation("remote", "add", "origin", "https://github.com/jedwards4b/fleximod-test2") gitp.git_operation("fetch", "origin", "main") + gitp.git_operation("checkout", "main") return test_dir @pytest.fixture @@ -130,6 +130,9 @@ def _run_fleximod(path, args, input=None): result = subprocess.run(cmd, cwd=path, input=input, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + if result.returncode: + print(result.stdout) + print(result.stderr) return result return _run_fleximod diff --git a/tests/test_c_required.py b/tests/test_c_required.py index d6cc040b8d..89ab8d294d 100644 --- a/tests/test_c_required.py +++ b/tests/test_c_required.py @@ -20,6 +20,10 @@ def test_required(git_fleximod, test_repo, shared_repos): assert result.returncode == 0 status = git_fleximod(test_repo, f"status {repo_name}") assert shared_repos["status3"] in status.stdout + status = git_fleximod(test_repo, f"update --optional") + assert result.returncode == 0 + status = git_fleximod(test_repo, f"status {repo_name}") + assert shared_repos["status4"] in status.stdout status = git_fleximod(test_repo, f"update {repo_name}") assert result.returncode == 0 status = git_fleximod(test_repo, f"status {repo_name}") diff --git a/tests/test_d_complex.py b/tests/test_d_complex.py index 2983b0193f..fdce516274 100644 --- a/tests/test_d_complex.py +++ b/tests/test_d_complex.py @@ -4,4 +4,64 @@ def test_complex_checkout(git_fleximod, complex_repo, logger): status = git_fleximod(complex_repo, "status") - print(status) + assert("ToplevelOptional not checked out, aligned at tag v5.3.2" in status.stdout) + assert("ToplevelRequired not checked out, aligned at tag MPIserial_2.5.0" in status.stdout) + assert("AlwaysRequired not checked out, aligned at tag MPIserial_2.4.0" in status.stdout) + assert("Complex not checked out, aligned at tag testtag01" in status.stdout) + assert("AlwaysOptional not checked out, aligned at tag MPIserial_2.3.0" in status.stdout) + + # This should checkout and update test_submodule and complex_sub + result = git_fleximod(complex_repo, "update") + assert result.returncode == 0 + + status = git_fleximod(complex_repo, "status") + assert("ToplevelOptional not checked out, aligned at tag v5.3.2" in status.stdout) + assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) + assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) + assert("Complex at tag testtag01" in status.stdout) + + # now check the complex_sub + root = (complex_repo / "modules" / "complex") + assert(not (root / "libraries" / "gptl" / ".git").exists()) + assert(not (root / "libraries" / "mpi-serial" / ".git").exists()) + assert((root / "modules" / "mpi-serial" / ".git").exists()) + assert(not (root / "modules" / "mpi-serial2" / ".git").exists()) + assert((root / "modules" / "mpi-sparse" / ".git").exists()) + assert((root / "modules" / "mpi-sparse" / "m4").exists()) + assert(not (root / "modules" / "mpi-sparse" / "README").exists()) + + # update a single optional submodule + + result = git_fleximod(complex_repo, "update ToplevelOptional") + assert result.returncode == 0 + + status = git_fleximod(complex_repo, "status") + assert("ToplevelOptional at tag v5.3.2" in status.stdout) + assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) + assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) + assert("Complex at tag testtag01" in status.stdout) + assert("AlwaysOptional not checked out, aligned at tag MPIserial_2.3.0" in status.stdout) + + + # Finally update optional + result = git_fleximod(complex_repo, "update --optional") + assert result.returncode == 0 + + status = git_fleximod(complex_repo, "status") + assert("ToplevelOptional at tag v5.3.2" in status.stdout) + assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) + assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) + assert("Complex at tag testtag01" in status.stdout) + assert("AlwaysOptional at tag MPIserial_2.3.0" in status.stdout) + + # now check the complex_sub + root = (complex_repo / "modules" / "complex" ) + assert(not (root / "libraries" / "gptl" / ".git").exists()) + assert(not (root / "libraries" / "mpi-serial" / ".git").exists()) + assert((root / "modules" / "mpi-serial" / ".git").exists()) + assert((root / "modules" / "mpi-serial2" / ".git").exists()) + assert((root / "modules" / "mpi-sparse" / ".git").exists()) + assert((root / "modules" / "mpi-sparse" / "m4").exists()) + assert(not (root / "modules" / "mpi-sparse" / "README").exists()) + + From 194d845ad2131c7ce50e1f5a14fc5476799c87fb Mon Sep 17 00:00:00 2001 From: James Edwards Date: Sat, 17 Feb 2024 08:08:57 -0700 Subject: [PATCH 8/8] add git user info to tests --- .github/workflows/pytest.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 4809a2787f..0868dd9a33 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -70,5 +70,8 @@ jobs: # And finally run tests. I'm using pytest and all my pytest config is in my `pyproject.toml` # so this line is super-simple. But it could be as complex as you need. - - run: poetry run pytest + - run: | + git config --global user.name "${GITHUB_ACTOR}" + git config --global user.email "${GITHUB_ACTOR_ID}+${GITHUB_ACTOR}@users.noreply.github.com" + poetry run pytest