diff --git a/CHANGELOG.md b/CHANGELOG.md index af2b01945..d849bcb39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,10 +41,14 @@ and this project adheres to [Semantic Versioning][semver]. ### Fixed +- Fix update status when a target repo was updated and the auth repo was not ([532]) +- Fix merge-commit which wasn't updating the remote-tracking branch ([532]) +- Fix removal of additional local commits ([532]) - Fix top-level authentication repository update to correctly update child auth repos ([528]) - Fix setup role when specifying public keys in keys-description ([511]) - `check_if_repositories_clean` error now returns a list of repositories which aren't clean, instead of a single repository ([525]) +[532]: https://github.com/openlawlibrary/taf/pull/532 [529]: https://github.com/openlawlibrary/taf/pull/529 [528]: https://github.com/openlawlibrary/taf/pull/528 [525]: https://github.com/openlawlibrary/taf/pull/525 diff --git a/taf/api/targets.py b/taf/api/targets.py index 34b59a954..6c1906e27 100644 --- a/taf/api/targets.py +++ b/taf/api/targets.py @@ -658,6 +658,11 @@ def _update_target_repos( return target_repo = GitRepository(path=target_repo_path) if target_repo.is_git_repository: + if target_repo.head_commit_sha() is None: + taf_logger.warning( + f"Repository {repo_path} does not have the HEAD reference" + ) + return data = {"commit": target_repo.head_commit_sha()} if add_branch: data["branch"] = target_repo.get_current_branch() diff --git a/taf/git.py b/taf/git.py index 3fa11dc30..644aef66c 100644 --- a/taf/git.py +++ b/taf/git.py @@ -371,6 +371,11 @@ def all_commits_on_branch( ) latest_commit_id = branch_obj.target else: + if self.head_commit_sha() is None: + raise GitError( + self, + message=f"Error occurred while getting commits of branch {branch}. No HEAD reference", + ) latest_commit_id = repo[repo.head.target].id sort = pygit2.GIT_SORT_REVERSE if reverse else pygit2.GIT_SORT_NONE @@ -392,7 +397,11 @@ def all_commits_since_commit( """ if since_commit is None: - return self.all_commits_on_branch(branch=branch, reverse=reverse) + try: + return self.all_commits_on_branch(branch=branch, reverse=reverse) + except GitError as e: + self._log_warning(str(e)) + return [] try: self.commit_exists(commit_sha=since_commit) @@ -407,6 +416,8 @@ def all_commits_since_commit( return [] latest_commit_id = branch_obj.target else: + if self.head_commit_sha() is None: + return [] latest_commit_id = repo[repo.head.target].id if repo.descendant_of(since_commit, latest_commit_id): @@ -817,7 +828,7 @@ def checkout_commit(self, commit: str) -> None: reraise_error=True, ) - def is_branch_with_unpushed_commits(self, branch_name): + def branch_unpushed_commits(self, branch_name): repo = self.pygit_repo local_branch = repo.branches.get(branch_name) @@ -849,7 +860,7 @@ def is_branch_with_unpushed_commits(self, branch_name): else: break - return bool(unpushed_commits) + return [commit.id for commit in unpushed_commits] def commit(self, message: str) -> str: self._git("add -A") @@ -1322,9 +1333,15 @@ def merge_commit( commit: str, fast_forward_only: Optional[bool] = False, check_if_merge_completed: Optional[bool] = False, + update_remote_tracking: Optional[bool] = True, ) -> bool: fast_forward_only_flag = "--ff-only" if fast_forward_only else "" self._git("merge {} {}", commit, fast_forward_only_flag, log_error=True) + if update_remote_tracking: + current_branch = self.get_current_branch() + self._git( + f"update-ref refs/remotes/origin/{current_branch} HEAD", log_error=True + ) # Update remote tracking if check_if_merge_completed: try: self._git("rev-parse -q --verify MERGE_HEAD") diff --git a/taf/tests/test_git/conftest.py b/taf/tests/test_git/conftest.py index a59b6f695..c328bf4c4 100644 --- a/taf/tests/test_git/conftest.py +++ b/taf/tests/test_git/conftest.py @@ -41,3 +41,14 @@ def clone_repository(): yield repo repo.cleanup() shutil.rmtree(path, onerror=on_rm_error) + + +@fixture +def empty_repository(): + path = TEST_DIR / CLONE_REPO_NAME + path.mkdir(exist_ok=True, parents=True) + repo = GitRepository(path=path) + repo.init_repo() + yield repo + repo.cleanup() + shutil.rmtree(path, onerror=on_rm_error) diff --git a/taf/tests/test_git/test_git.py b/taf/tests/test_git/test_git.py index be2145ddb..7968e1407 100644 --- a/taf/tests/test_git/test_git.py +++ b/taf/tests/test_git/test_git.py @@ -11,14 +11,14 @@ def test_clone_from_local(repository, clone_repository): assert len(commits) -def test_is_branch_with_unpushed_commits(repository, clone_repository): +def test_branch_unpushed_commits(repository, clone_repository): clone_repository.clone_from_disk(repository.path, keep_remote=True) branch = clone_repository.branches()[0] clone_repository.reset_num_of_commits(1, True) - assert not clone_repository.is_branch_with_unpushed_commits(branch) + assert not len(clone_repository.branch_unpushed_commits(branch)) (clone_repository.path / "test3.txt").write_text("Updated test3") clone_repository.commit(message="Update test3.txt") - assert clone_repository.is_branch_with_unpushed_commits(branch) + assert len(clone_repository.branch_unpushed_commits(branch)) def test_is_git_repository_root_bare(repository): @@ -41,3 +41,9 @@ def test_head_commit_sha(): match=f"Repo {repo.name}: The path '{repo.path.as_posix()}' is not a Git repository.", ): repo.head_commit_sha() is not None + + +def test_all_commits_since_commit_when_repo_empty(empty_repository): + all_commits_empty = empty_repository.all_commits_since_commit() + assert isinstance(all_commits_empty, list) + assert len(all_commits_empty) == 0 diff --git a/taf/tests/test_updater/conftest.py b/taf/tests/test_updater/conftest.py index 71e95fb63..37f27a72a 100644 --- a/taf/tests/test_updater/conftest.py +++ b/taf/tests/test_updater/conftest.py @@ -173,9 +173,15 @@ def execute_tasks(self): class RepositoryConfig: - def __init__(self, name: str, allow_unauthenticated_commits: bool = False): + def __init__( + self, + name: str, + allow_unauthenticated_commits: bool = False, + is_empty: bool = False, + ): self.name = name self.allow_unauthenticated_commits = allow_unauthenticated_commits + self.is_empty = is_empty @pytest.fixture @@ -209,6 +215,7 @@ def origin_auth_repo(request, test_name: str, origin_dir: Path): RepositoryConfig( f"{test_name}/{targets_config['name']}", targets_config.get("allow_unauthenticated_commits", False), + targets_config.get("is_empty", False), ) for targets_config in targets_config_list ] @@ -332,6 +339,8 @@ def _init_auth_repo( def initialize_git_repo(library_dir: Path, repo_name: str) -> GitRepository: repo_path = Path(library_dir, repo_name) + if repo_path.is_dir(): + shutil.rmtree(repo_path, onerror=on_rm_error) repo_path.mkdir(parents=True, exist_ok=True) repo = GitRepository(path=repo_path) repo.init_repo() @@ -349,10 +358,11 @@ def initialize_target_repositories( else: target_repo = GitRepository(library_dir, target_config.name) # create some files, content of these repositories is not important - for i in range(1, 3): - random_text = _generate_random_text() - (target_repo.path / f"test{i}.txt").write_text(random_text) - target_repo.commit("Initial commit") + if not target_config.is_empty: + for i in range(1, 3): + random_text = _generate_random_text() + (target_repo.path / f"test{i}.txt").write_text(random_text) + target_repo.commit("Initial commit") def sign_target_repositories(library_dir: Path, repo_name: str, keystore: Path): @@ -478,8 +488,12 @@ def setup_repository_no_target_repositories( return AuthenticationRepository(origin_dir, repo_name) -def add_valid_target_commits(auth_repo: AuthenticationRepository, target_repos: list): +def add_valid_target_commits( + auth_repo: AuthenticationRepository, target_repos: list, add_if_empty: bool = True +): for target_repo in target_repos: + if not add_if_empty and target_repo.head_commit_sha() is None: + continue update_target_files(target_repo, "Update target files") sign_target_repositories(TEST_DATA_ORIGIN_PATH, auth_repo.name, KEYSTORE_PATH) @@ -495,6 +509,17 @@ def add_unauthenticated_commits_to_all_target_repos(target_repos: list): update_target_files(target_repo, "Update target files") +def add_unauthenticated_commit_to_target_repo(target_repos: list, target_name: str): + for target_repo in target_repos: + if target_name in target_repo.name: + update_target_files(target_repo, "Update target files") + + +def add_unauthenticated_commits_to_target_repo(target_repos: list): + for target_repo in target_repos: + update_target_files(target_repo, "Update target files") + + def create_new_target_orphan_branches( auth_repo: AuthenticationRepository, target_repos: list, branch_name: str ): @@ -598,7 +623,7 @@ def update_role_metadata_invalid_signature( content["signatures"][0]["sign"] = "invalid signature" version = content["signed"]["version"] content["signed"]["version"] = version + 1 - role_metadata_path.write_text(json.dumps(content)) + role_metadata_path.write_text(json.dumps(content, indent=4)) auth_repo.commit("Invalid metadata update") @@ -631,11 +656,17 @@ def update_and_sign_metadata_without_clean_check( def update_target_files(target_repo: GitRepository, commit_message: str): text_to_add = _generate_random_text() # Iterate over all files in the repository directory + is_empty = True for file_path in target_repo.path.iterdir(): if file_path.is_file(): + is_empty = False existing_content = file_path.read_text(encoding="utf-8") new_content = existing_content + "\n" + text_to_add file_path.write_text(new_content, encoding="utf-8") + + if is_empty: + random_text = _generate_random_text() + (target_repo.path / "test.txt").write_text(random_text) target_repo.commit(commit_message) @@ -660,8 +691,6 @@ def add_file_without_commit(repo_path: str, filename: str): def remove_commits( - auth_repo: AuthenticationRepository, - target_repos: list, repo_path: str, num_commits: int = 1, ): diff --git a/taf/tests/test_updater/test_clone/test_clone_valid.py b/taf/tests/test_updater/test_clone/test_clone_valid.py index 36b21d8b4..5ee633385 100644 --- a/taf/tests/test_updater/test_clone/test_clone_valid.py +++ b/taf/tests/test_updater/test_clone/test_clone_valid.py @@ -1,6 +1,7 @@ import pytest from taf.tests.test_updater.conftest import ( SetupManager, + add_unauthenticated_commit_to_target_repo, add_unauthenticated_commits_to_all_target_repos, add_valid_target_commits, add_valid_unauthenticated_commits, @@ -13,6 +14,7 @@ from taf.tests.test_updater.update_utils import ( clone_client_target_repos_without_updater, update_and_check_commit_shas, + verify_repos_eixst, ) from taf.updater.types.update import OperationType, UpdateType @@ -316,3 +318,55 @@ def test_clone_valid_when_no_upstream_top_commits_unsigned( expected_repo_type=UpdateType.EITHER, no_upstream=True, ) + + +@pytest.mark.parametrize( + "origin_auth_repo", + [ + { + "targets_config": [ + {"name": "notempty"}, + {"name": "empty", "is_empty": True}, + ], + }, + ], + indirect=True, +) +def test_clone_when_target_empty(origin_auth_repo, client_dir): + + update_and_check_commit_shas( + OperationType.CLONE, + origin_auth_repo, + client_dir, + expected_repo_type=UpdateType.EITHER, + ) + verify_repos_eixst(client_dir, origin_auth_repo, exists=["notempty"]) + + +@pytest.mark.parametrize( + "origin_auth_repo", + [ + { + "targets_config": [ + {"name": "target1"}, + {"name": "target2", "is_empty": True}, + ], + }, + ], + indirect=True, +) +def test_clone_when_no_target_file_and_commit(origin_auth_repo, client_dir): + + setup_manager = SetupManager(origin_auth_repo) + setup_manager.add_task( + add_unauthenticated_commit_to_target_repo, kwargs={"target_name": "target2"} + ) + setup_manager.execute_tasks() + + update_and_check_commit_shas( + OperationType.CLONE, + origin_auth_repo, + client_dir, + expected_repo_type=UpdateType.EITHER, + ) + verify_repos_eixst(client_dir, origin_auth_repo, exists=["target1"]) diff --git a/taf/tests/test_updater/test_update/test_update_invalid.py b/taf/tests/test_updater/test_update/test_update_invalid.py index 32f0cd6f3..fae843260 100644 --- a/taf/tests/test_updater/test_update/test_update_invalid.py +++ b/taf/tests/test_updater/test_update/test_update_invalid.py @@ -1,6 +1,7 @@ import pytest from taf.auth_repo import AuthenticationRepository from taf.tests.test_updater.conftest import ( + INVALID_KEYS_PATTERN, TARGET_MISSMATCH_PATTERN, FORCED_UPDATE_PATTERN, UNCOMMITTED_CHANGES, @@ -10,9 +11,12 @@ add_unauthenticated_commits_to_all_target_repos, add_valid_target_commits, create_index_lock, + update_expiration_dates, update_file_without_commit, + update_role_metadata_invalid_signature, ) from taf.tests.test_updater.update_utils import ( + check_if_last_validated_commit_exists, clone_repositories, update_invalid_repos_and_check_if_repos_exist, ) @@ -237,3 +241,35 @@ def test_update_with_invalid_last_validated_commit(origin_auth_repo, client_dir) COMMIT_NOT_FOUND_PATTERN, True, ) + + +@pytest.mark.parametrize( + "origin_auth_repo", + [ + { + "targets_config": [{"name": "target1"}, {"name": "target2"}], + } + ], + indirect=True, +) +def test_update_invalid_target_invalid_singature(origin_auth_repo, client_dir): + + clone_repositories(origin_auth_repo, client_dir) + + setup_manager = SetupManager(origin_auth_repo) + setup_manager.add_task( + update_role_metadata_invalid_signature, kwargs={"role": "targets"} + ) + setup_manager.add_task(update_expiration_dates) + setup_manager.execute_tasks() + + update_invalid_repos_and_check_if_repos_exist( + OperationType.UPDATE, + origin_auth_repo, + client_dir, + INVALID_KEYS_PATTERN, + True, + ) + client_auth_repo = AuthenticationRepository(client_dir, origin_auth_repo.name) + # make sure that the last validated commit does not exist + check_if_last_validated_commit_exists(client_auth_repo, True) diff --git a/taf/tests/test_updater/test_update/test_update_valid.py b/taf/tests/test_updater/test_update/test_update_valid.py index 478fa0d77..b4e0e9d80 100644 --- a/taf/tests/test_updater/test_update/test_update_valid.py +++ b/taf/tests/test_updater/test_update/test_update_valid.py @@ -23,6 +23,7 @@ clone_repositories, load_target_repositories, update_and_check_commit_shas, + verify_repos_eixst, ) from taf.updater.types.update import OperationType, UpdateType @@ -656,10 +657,7 @@ def test_update_with_removed_commits_in_auth_repo(origin_auth_repo, client_dir): setup_manager.execute_tasks() client_auth_repo = AuthenticationRepository(client_dir, origin_auth_repo.name) - client_target_repos = load_target_repositories(client_auth_repo) remove_commits( - auth_repo=client_auth_repo, - target_repos=client_target_repos, repo_path=str(client_auth_repo.path), num_commits=1, ) @@ -693,10 +691,7 @@ def test_update_with_last_validated_commit_not_in_local_repo( client_auth_repo = AuthenticationRepository(client_dir, origin_auth_repo.name) client_auth_repo.set_last_validated_commit(origin_top_commit_sha) - client_target_repos = load_target_repositories(client_auth_repo) remove_commits( - auth_repo=client_auth_repo, - target_repos=client_target_repos, repo_path=str(client_auth_repo.path), num_commits=1, ) @@ -729,3 +724,73 @@ def test_update_with_no_last_validated_commit(origin_auth_repo, client_dir): last_validated_commit_file.unlink() # Remove the file update_and_check_commit_shas(OperationType.UPDATE, origin_auth_repo, client_dir) + + +@pytest.mark.parametrize( + "origin_auth_repo", + [ + { + "targets_config": [ + {"name": "notempty"}, + {"name": "empty", "is_empty": True}, + ], + }, + ], + indirect=True, +) +def test_update_when_target_empty(origin_auth_repo, client_dir): + + clone_repositories(origin_auth_repo, client_dir) + + setup_manager = SetupManager(origin_auth_repo) + setup_manager.add_task(add_valid_target_commits, kwargs={"add_if_empty": False}) + setup_manager.execute_tasks() + + update_and_check_commit_shas( + OperationType.UPDATE, + origin_auth_repo, + client_dir, + skip_check_last_validated=True, + ) + verify_repos_eixst(client_dir, origin_auth_repo, exists=["notempty"]) + + +@pytest.mark.parametrize( + "origin_auth_repo", + [ + { + "targets_config": [{"name": "target1"}, {"name": "target2"}], + }, + ], + indirect=True, +) +def test_update_valid_when_several_updates(origin_auth_repo, client_dir): + clone_repositories(origin_auth_repo, client_dir) + + setup_manager = SetupManager(origin_auth_repo) + setup_manager.add_task(add_valid_target_commits) + setup_manager.add_task(add_valid_target_commits) + setup_manager.add_task(add_valid_target_commits) + setup_manager.execute_tasks() + + update_and_check_commit_shas( + OperationType.UPDATE, + origin_auth_repo, + client_dir, + skip_check_last_validated=True, + ) + client_auth_repo = AuthenticationRepository(client_dir, origin_auth_repo.name) + client_target_repos = load_target_repositories(client_auth_repo) + setup_manager = SetupManager(client_auth_repo) + for target_repo in client_target_repos.values(): + setup_manager.add_task( + remove_commits, kwargs={"repo_path": target_repo.path, "num_commits": 1} + ) + break + setup_manager.execute_tasks() + update_and_check_commit_shas( + OperationType.UPDATE, + origin_auth_repo, + client_dir, + skip_check_last_validated=True, + ) diff --git a/taf/tests/test_updater/test_update/test_updater_output.py b/taf/tests/test_updater/test_update/test_updater_output.py new file mode 100644 index 000000000..56832ac22 --- /dev/null +++ b/taf/tests/test_updater/test_update/test_updater_output.py @@ -0,0 +1,63 @@ +import pytest +from taf.auth_repo import AuthenticationRepository +from taf.tests.test_updater.conftest import ( + SetupManager, + add_valid_target_commits, + remove_commits, +) +from taf.tests.test_updater.update_utils import ( + clone_repositories, + load_target_repositories, + update_and_check_commit_shas, +) +from taf.updater.types.update import OperationType + + +@pytest.mark.parametrize( + "origin_auth_repo", + [ + { + "targets_config": [{"name": "target1"}, {"name": "target2"}], + }, + ], + indirect=True, +) +def test_update_valid_when_several_updates(origin_auth_repo, client_dir): + clone_repositories(origin_auth_repo, client_dir) + + setup_manager = SetupManager(origin_auth_repo) + setup_manager.add_task(add_valid_target_commits) + setup_manager.add_task(add_valid_target_commits) + setup_manager.add_task(add_valid_target_commits) + setup_manager.execute_tasks() + + update_output = update_and_check_commit_shas( + OperationType.UPDATE, + origin_auth_repo, + client_dir, + skip_check_last_validated=True, + ) + assert update_output["changed"] + client_auth_repo = AuthenticationRepository(client_dir, origin_auth_repo.name) + client_target_repos = load_target_repositories(client_auth_repo) + setup_manager = SetupManager(client_auth_repo) + for target_repo in client_target_repos.values(): + setup_manager.add_task( + remove_commits, kwargs={"repo_path": target_repo.path, "num_commits": 1} + ) + break + setup_manager.execute_tasks() + update_output = update_and_check_commit_shas( + OperationType.UPDATE, + origin_auth_repo, + client_dir, + skip_check_last_validated=True, + ) + assert update_output["changed"] + update_output = update_and_check_commit_shas( + OperationType.UPDATE, + origin_auth_repo, + client_dir, + skip_check_last_validated=True, + ) + assert not update_output["changed"] diff --git a/taf/tests/test_updater/test_update/test_validation_and_sync.py b/taf/tests/test_updater/test_update/test_validation_and_sync.py index 99beea9b6..0263c7927 100644 --- a/taf/tests/test_updater/test_update/test_validation_and_sync.py +++ b/taf/tests/test_updater/test_update/test_validation_and_sync.py @@ -265,7 +265,6 @@ def test_mixed_target_repo_states(origin_auth_repo, client_dir): updated_repo = client_target_repos[1] # target2 old_commit = reverted_repo.head_commit_sha() - # Add valid commits first to setup_manager.add_task(add_valid_target_commits) setup_manager.add_task( @@ -289,3 +288,40 @@ def test_mixed_target_repo_states(origin_auth_repo, client_dir): ) verify_client_repos_state(client_dir, origin_auth_repo) + + +@pytest.mark.parametrize( + "origin_auth_repo", + [ + { + "targets_config": [ + {"name": "target1", "allow_unauthenticated_commits": True}, + {"name": "target2", "allow_unauthenticated_commits": True}, + ], + } + ], + indirect=True, +) +def test_update_when_unauthenticated_allowed_different_commits_on_remote( + origin_auth_repo, client_dir +): + clone_repositories( + origin_auth_repo, + client_dir, + ) + setup_manager = SetupManager(origin_auth_repo) + setup_manager.add_task(add_unauthenticated_commits_to_all_target_repos) + setup_manager.execute_tasks() + + client_auth_repo = AuthenticationRepository(client_dir, origin_auth_repo.name) + setup_manager = SetupManager(client_auth_repo) + setup_manager.add_task(add_unauthenticated_commits_to_all_target_repos) + setup_manager.execute_tasks() + + update_and_check_commit_shas( + OperationType.UPDATE, + origin_auth_repo, + client_dir, + force=True, + num_of_commits_to_remove=1, + ) diff --git a/taf/tests/test_updater/update_utils.py b/taf/tests/test_updater/update_utils.py index b0f74d094..3b49db2b2 100644 --- a/taf/tests/test_updater/update_utils.py +++ b/taf/tests/test_updater/update_utils.py @@ -1,4 +1,5 @@ import os +import shutil import pytest from pathlib import Path from freezegun import freeze_time @@ -118,6 +119,8 @@ def clone_repositories( excluded_target_globs=None, ): + if clients_dir.is_dir(): + shutil.rmtree(clients_dir) config = UpdateConfig( operation=OperationType.CLONE, url=str(origin_auth_repo.path), @@ -159,19 +162,29 @@ def _get_valid_update_time(origin_auth_repo_path): return datetime.strptime(expires, "%Y-%m-%dT%H:%M:%SZ").date().strftime("%Y-%m-%d") -def _get_head_commit_shas(client_repos): +def _get_head_commit_shas(client_repos, num_of_commits_to_remove=0): start_head_shas = defaultdict(dict) if client_repos is not None: for repo_rel_path, repo in client_repos.items(): for branch in repo.branches(): - start_head_shas[repo_rel_path][branch] = repo.top_commit_of_branch( - branch - ) + if not num_of_commits_to_remove: + start_head_shas[repo_rel_path][branch] = repo.top_commit_of_branch( + branch + ) + else: + all_commits = repo.all_commits_on_branch(branch) + start_head_shas[repo_rel_path][branch] = all_commits[ + -num_of_commits_to_remove - 1 + ] return start_head_shas def load_target_repositories( - auth_repo, library_dir=None, excluded_target_globs=None, commits=None + auth_repo, + library_dir=None, + excluded_target_globs=None, + commits=None, + only_load_targets=False, ): if library_dir is None: library_dir = auth_repo.path.parent.parent @@ -179,7 +192,7 @@ def load_target_repositories( repositoriesdb.load_repositories( auth_repo, library_dir=library_dir, - only_load_targets=True, + only_load_targets=only_load_targets, excluded_target_globs=excluded_target_globs, commits=commits, ) @@ -200,6 +213,7 @@ def update_and_check_commit_shas( bare=False, no_upstream=False, skip_check_last_validated=False, + num_of_commits_to_remove=0, ): client_repos = load_target_repositories(origin_auth_repo, clients_dir) client_repos = { @@ -212,7 +226,7 @@ def update_and_check_commit_shas( clients_auth_repo = GitRepository(path=clients_auth_repo_path) if clients_auth_repo_path.is_dir(): client_repos[clients_auth_repo.name] = clients_auth_repo - start_head_shas = _get_head_commit_shas(client_repos) + start_head_shas = _get_head_commit_shas(client_repos, num_of_commits_to_remove) config = UpdateConfig( operation=operation, @@ -228,9 +242,9 @@ def update_and_check_commit_shas( ) if operation == OperationType.CLONE: - clone_repository(config) + update_ret = clone_repository(config) else: - update_repository(config) + update_ret = update_repository(config) origin_root_dir = origin_auth_repo.path.parent.parent check_if_commits_match( @@ -252,6 +266,7 @@ def update_and_check_commit_shas( if fnmatch.fnmatch(target_repo.name, excluded_target_glob): assert not target_repo.path.is_dir() break + return update_ret def update_invalid_repos_and_check_if_repos_exist( @@ -307,6 +322,32 @@ def _update_expect_error(): assert not client_repository.path.exists() +def verify_repos_eixst( + client_dir: Path, origin_auth_repo: AuthenticationRepository, exists: list +): + client_auth_repo = AuthenticationRepository(path=client_dir / origin_auth_repo.name) + client_target_repos = load_target_repositories( + client_auth_repo, library_dir=client_dir + ) + for repo in client_target_repos.values(): + if repo.name.split("/")[-1] in exists: + assert repo.is_git_repository + else: + assert not repo.path.is_dir() + + +def verify_repo_empty( + client_dir: Path, origin_auth_repo: AuthenticationRepository, target_name_part: str +): + client_auth_repo = AuthenticationRepository(path=client_dir / origin_auth_repo.name) + client_target_repos = load_target_repositories( + client_auth_repo, library_dir=client_dir + ) + for name, repo in client_target_repos.items(): + if target_name_part in name: + assert not len(repo.all_commits_on_branch()) + + def verify_client_repos_state( client_dir: Path, origin_auth_repo: AuthenticationRepository ): diff --git a/taf/updater/updater_pipeline.py b/taf/updater/updater_pipeline.py index cd7c3bb79..b5f13b6d1 100644 --- a/taf/updater/updater_pipeline.py +++ b/taf/updater/updater_pipeline.py @@ -459,17 +459,18 @@ def check_if_local_repositories_clean(self): ) dirty_index_repos.append(auth_repo.name) - if auth_repo.is_branch_with_unpushed_commits(auth_repo.default_branch): + unpushed_commits = auth_repo.branch_unpushed_commits( + auth_repo.default_branch + ) + if unpushed_commits: if self.force: taf_logger.info( f"Resetting repository {auth_repo.name} to clean state for a forced update." ) - auth_repo.clean_and_reset() - last_remote_commit = auth_repo.get_last_remote_commit( - auth_repo.urls[0] + _remove_unpushed_commits( + auth_repo, auth_repo.default_branch, unpushed_commits ) - if last_remote_commit: - auth_repo.reset_to_commit(last_remote_commit, hard=True) + else: unpushed_commits_repos_and_branches.append( (auth_repo.name, auth_repo.default_branch) @@ -495,12 +496,15 @@ def check_if_local_repositories_clean(self): target = auth_repo.get_target(repository.name) if target and "branch" in target: branch = target["branch"] - if repository.is_branch_with_unpushed_commits(branch): + unpushed_commits = repository.branch_unpushed_commits(branch) + if unpushed_commits: if self.force: taf_logger.info( f"Resetting repository {repository.name} to clean state for a forced update." ) - repository.clean_and_reset() + _remove_unpushed_commits( + repository, branch, unpushed_commits + ) else: unpushed_commits_repos_and_branches.append( (repository.name, branch) @@ -513,7 +517,6 @@ def check_if_local_repositories_clean(self): raise MultipleRepositoriesNotCleanError( dirty_index_repos, unpushed_commits_repos_and_branches ) - return UpdateStatus.SUCCESS except Exception as e: self.state.errors.append(e) @@ -1111,7 +1114,8 @@ def check_if_local_target_repositories_clean(self): for branch in self.state.target_branches_data_from_auth_repo[ repository.name ]: - if repository.is_branch_with_unpushed_commits(branch): + + if repository.branch_unpushed_commits(branch): unpushed_commits_repos_and_branches_error.append( (repository.name, branch) ) @@ -1402,6 +1406,7 @@ def merge_commits(self): taf_logger.info( f"{self.state.auth_repo_name}: Merging commits into target repositories..." ) + events_list = [] try: if self.only_validate: return self.state.update_status @@ -1417,9 +1422,14 @@ def merge_commits(self): ].items(): last_validated_commit = validated_commits[-1] commit_to_merge = last_validated_commit - _merge_commit( + update_status = _merge_commit( repository, branch, commit_to_merge, force_revert=True ) + events_list.append(update_status) + + if self.state.event == Event.UNCHANGED and Event.CHANGED in events_list: + # the auth repository was not updated, but one of the target repositories was + self.state.event = Event.CHANGED return self.state.update_status except Exception as e: self.state.errors.append(e) @@ -1606,6 +1616,12 @@ def _is_unauthenticated_allowed(repository): return repository.custom.get("allow-unauthenticated-commits", False) +def _remove_unpushed_commits(repository, branch, unpushed_commits): + repository.clean_and_reset() + repository.checkout_branch(branch) + repository.reset_num_of_commits(len(unpushed_commits), hard=True) + + def _run_tuf_updater(git_fetcher, auth_repo_name): auth_repo_name = auth_repo_name or "" taf_logger.info(f"{auth_repo_name}: Running TUF validation...") @@ -1778,12 +1794,11 @@ def _merge_commit(repository, branch, commit_to_merge, force_revert=True): ) if repository.top_commit_of_branch(branch) == commit_to_merge: - return + return Event.UNCHANGED commits_since_to_merge = repository.all_commits_since_commit( commit_to_merge, branch=branch ) - if not len(commits_since_to_merge): taf_logger.info( "{} Merging commit {} into branch {}", @@ -1808,3 +1823,4 @@ def _merge_commit(repository, branch, commit_to_merge, force_revert=True): format_commit(commit_to_merge), branch, ) + return Event.CHANGED