From 37c18bbc2f7366d9dc1f9a6a786d6ccb7fa4fa01 Mon Sep 17 00:00:00 2001 From: Renata Date: Fri, 27 Sep 2024 15:05:47 -0400 Subject: [PATCH] fear: work on detecting cases when lvc is invalid --- taf/tests/test_updater/conftest.py | 4 +- .../test_update_different_states.py | 561 ++++++++++++++++++ .../test_update/test_update_invalid.py | 145 +---- .../test_update/test_update_partial.py | 10 +- taf/updater/updater_pipeline.py | 115 ++-- 5 files changed, 641 insertions(+), 194 deletions(-) create mode 100644 taf/tests/test_updater/test_update/test_update_different_states.py diff --git a/taf/tests/test_updater/conftest.py b/taf/tests/test_updater/conftest.py index 924f96d5..0578415c 100644 --- a/taf/tests/test_updater/conftest.py +++ b/taf/tests/test_updater/conftest.py @@ -63,6 +63,9 @@ UNCOMMITTED_CHANGES = r"Update of (\w+\/\w+) failed due to error: Repository (\w+\/\w+) should contain only committed changes\. \nPlease update the repository at (.+) manually and try again\." UPDATE_ERROR_PATTERN = r"Update of (\w+\/\w+) failed due to error: Validation of authentication repository (\w+\/\w+) failed at revision ([0-9a-f]+) due to error: .*" FORCED_UPDATE_PATTERN = r"Update of (\w+\/\w+) failed due to error: Repositories ([\w/,\s-]+) have uncommitted changes. Commit and push or revert the changes and run the command again." +BEHIND_LVC_PATTERN = r"Update of (\w+\/\w+) failed due to error: Top commit of repository \1 ([0-9a-f]{40}) is not equal to or newer than the last successful commit." +LVC_NOT_IN_REPO_PATTERN = r"Update of (\w+\/\w+) failed due to error: \1: Last validated commit (\w{40}) is not in repository \1\s*Run the updater with the --force flag to run the validation from the first commit" +LVC_NOT_IN_REMOTE_PATTERN = r"Update of ([\w_\/]+) failed due to error: Last validated commit ([\da-f]{40}) is no longer on (\w+) of the remote ([\w_\/]+) repository.*" UNPUSHED_COMMITS_PATTERN = r"Update of (\w+\/\w+) failed due to error:\s*\nThe following repository has unpushed commits on branches: ([\w\/]+): \(([\w,-]+)\)" REMOVED_COMMITS_PATTERN = r"Update of (\w+/\w+) failed due to error: Last validated commit ([0-9a-f]{40}) is not in the remote repository." INVALID_TIMESTAMP_PATTERN = r"^Update of (\w+\/\w+) failed due to error: Update of (\w+\/\w+) failed. One or more referenced authentication repositories could not be validated:\n Validation of authentication repository (\w+\/\w+) failed at revision ([0-9a-f]{40}) due to error: timestamp was signed by (\d+)\/(\d+) keys$" @@ -74,7 +77,6 @@ r"([0-9a-f]{40}) committed on (\d{4}-\d{2}-\d{2}): data repository (\w+\/\w+) was " r"supposed to be at commit ([0-9a-f]{40}) but (repo was at|commit not on branch) (\w+)" ) -COMMIT_NOT_FOUND_PATTERN = r"Update of (\w+\/\w+) failed due to error: object not found - no match for id \((\w{40})\)" # Disable console logging for all tests diff --git a/taf/tests/test_updater/test_update/test_update_different_states.py b/taf/tests/test_updater/test_update/test_update_different_states.py new file mode 100644 index 00000000..6251a1ba --- /dev/null +++ b/taf/tests/test_updater/test_update/test_update_different_states.py @@ -0,0 +1,561 @@ +import pytest +from taf.auth_repo import AuthenticationRepository +from taf.git import GitRepository +from taf.tests.test_updater.conftest import ( + BEHIND_LVC_PATTERN, + FORCED_UPDATE_PATTERN, + LVC_NOT_IN_REPO_PATTERN, + UNPUSHED_COMMITS_PATTERN, + SetupManager, + add_file_to_auth_repo_without_committing, + add_file_to_target_repo_without_committing, + add_unauthenticated_commit_to_target_repo, + add_valid_target_commits, + remove_commits, + update_auth_repo_without_committing, + update_expiration_dates, + update_target_repo_without_committing, +) +from taf.tests.test_updater.update_utils import ( + clone_repositories, + update_and_check_commit_shas, + update_invalid_repos_and_check_if_repos_exist, +) +from taf.updater.types.update import OperationType + + +@pytest.mark.parametrize( + "origin_auth_repo", + [ + { + "targets_config": [{"name": "target1"}, {"name": "target2"}], + }, + ], + indirect=True, +) +def test_update_valid_dirty_index_auth_repo(origin_auth_repo, client_dir): + clone_repositories( + origin_auth_repo, + client_dir, + ) + + client_auth_repo_path = client_dir / origin_auth_repo.name + client_auth_repo = AuthenticationRepository(path=client_auth_repo_path) + + setup_manager = SetupManager(client_auth_repo) + setup_manager.add_task(update_auth_repo_without_committing) + setup_manager.execute_tasks() + assert client_auth_repo.something_to_commit() + + # the update should fail without the force flag + update_invalid_repos_and_check_if_repos_exist( + OperationType.UPDATE, + origin_auth_repo, + client_dir, + FORCED_UPDATE_PATTERN, + True, + ) + + # now call with the force flag + update_and_check_commit_shas( + OperationType.UPDATE, + origin_auth_repo, + client_dir, + force=True, + ) + + assert not client_auth_repo.something_to_commit() + + setup_manager.add_task(add_file_to_auth_repo_without_committing) + setup_manager.execute_tasks() + + assert client_auth_repo.something_to_commit() + + # the update should fail without the force flag + update_invalid_repos_and_check_if_repos_exist( + OperationType.UPDATE, + origin_auth_repo, + client_dir, + FORCED_UPDATE_PATTERN, + True, + ) + + update_and_check_commit_shas( + OperationType.UPDATE, + origin_auth_repo, + client_dir, + force=True, + ) + assert not client_auth_repo.something_to_commit() + + +@pytest.mark.parametrize( + "origin_auth_repo", + [ + { + "targets_config": [{"name": "target1"}, {"name": "target2"}], + }, + ], + indirect=True, +) +def test_update_valid_dirty_index_target_repo(origin_auth_repo, client_dir): + clone_repositories( + origin_auth_repo, + client_dir, + ) + client_auth_repo_path = client_dir / origin_auth_repo.name + client_auth_repo = AuthenticationRepository(path=client_auth_repo_path) + + setup_manager = SetupManager(client_auth_repo) + setup_manager.add_task( + update_target_repo_without_committing, kwargs={"target_name": "target1"} + ) + setup_manager.execute_tasks() + + target1 = GitRepository(path=(client_auth_repo_path.parent / "target1")) + assert target1.something_to_commit() + + # the update should fail without the force flag + update_invalid_repos_and_check_if_repos_exist( + OperationType.UPDATE, + origin_auth_repo, + client_dir, + FORCED_UPDATE_PATTERN, + True, + ) + + # now call with the force flag + update_and_check_commit_shas( + OperationType.UPDATE, + origin_auth_repo, + client_dir, + force=True, + ) + + assert not target1.something_to_commit() + + setup_manager.add_task( + add_file_to_target_repo_without_committing, kwargs={"target_name": "target1"} + ) + setup_manager.execute_tasks() + + assert target1.something_to_commit() + + # the update should fail without the force flag + update_invalid_repos_and_check_if_repos_exist( + OperationType.UPDATE, + origin_auth_repo, + client_dir, + FORCED_UPDATE_PATTERN, + True, + ) + + update_and_check_commit_shas( + OperationType.UPDATE, + origin_auth_repo, + client_dir, + force=True, + ) + assert not target1.something_to_commit() + + +@pytest.mark.parametrize( + "origin_auth_repo", + [ + { + "targets_config": [{"name": "target1"}, {"name": "target2"}], + }, + ], + indirect=True, +) +def test_update_unpushed_commits_auth_repo(origin_auth_repo, client_dir): + clone_repositories( + origin_auth_repo, + client_dir, + ) + + client_auth_repo_path = client_dir / origin_auth_repo.name + client_auth_repo = AuthenticationRepository(path=client_auth_repo_path) + + setup_manager = SetupManager(client_auth_repo) + setup_manager.add_task(update_expiration_dates, kwargs={"push": False}) + setup_manager.execute_tasks() + assert len( + client_auth_repo.branch_unpushed_commits(client_auth_repo.default_branch) + ) + + # the update should fail without the force flag + update_invalid_repos_and_check_if_repos_exist( + OperationType.UPDATE, + origin_auth_repo, + client_dir, + UNPUSHED_COMMITS_PATTERN, + True, + ) + + num_of_commits_to_remove = {client_auth_repo.name: 1} + # now call with the force flag + update_and_check_commit_shas( + OperationType.UPDATE, + origin_auth_repo, + client_dir, + force=True, + num_of_commits_to_remove=num_of_commits_to_remove, + ) + + assert not len( + client_auth_repo.branch_unpushed_commits(client_auth_repo.default_branch) + ) + + +@pytest.mark.parametrize( + "origin_auth_repo", + [ + { + "targets_config": [{"name": "target1"}, {"name": "target2"}], + }, + ], + indirect=True, +) +def test_update_unpushed_commits_auth_repo(origin_auth_repo, client_dir): + clone_repositories( + origin_auth_repo, + client_dir, + ) + + client_auth_repo_path = client_dir / origin_auth_repo.name + client_auth_repo = AuthenticationRepository(path=client_auth_repo_path) + client_target_repo_path = client_auth_repo_path.parent / "target1" + client_target_repo = GitRepository(path=client_target_repo_path) + + setup_manager = SetupManager(client_auth_repo) + setup_manager.add_task( + add_unauthenticated_commit_to_target_repo, + kwargs={"target_name": client_target_repo.name}, + ) + setup_manager.execute_tasks() + assert len( + client_target_repo.branch_unpushed_commits(client_target_repo.default_branch) + ) + + # the update should fail without the force flag + update_invalid_repos_and_check_if_repos_exist( + OperationType.UPDATE, + origin_auth_repo, + client_dir, + UNPUSHED_COMMITS_PATTERN, + True, + ) + + num_of_commits_to_remove = {client_target_repo.name: 1} + # now call with the force flag + update_and_check_commit_shas( + OperationType.UPDATE, + origin_auth_repo, + client_dir, + force=True, + num_of_commits_to_remove=num_of_commits_to_remove, + ) + + assert not len( + client_target_repo.branch_unpushed_commits(client_target_repo.default_branch) + ) + + +@pytest.mark.parametrize( + "origin_auth_repo", + [ + { + "targets_config": [{"name": "target1"}, {"name": "target2"}], + }, + ], + indirect=True, +) +def test_update_valid_dirty_index_target_repo(origin_auth_repo, client_dir): + clone_repositories( + origin_auth_repo, + client_dir, + ) + client_auth_repo_path = client_dir / origin_auth_repo.name + client_auth_repo = AuthenticationRepository(path=client_auth_repo_path) + + setup_manager = SetupManager(client_auth_repo) + setup_manager.add_task( + update_target_repo_without_committing, kwargs={"target_name": "target1"} + ) + setup_manager.execute_tasks() + + target1 = GitRepository(path=(client_auth_repo_path.parent / "target1")) + assert target1.something_to_commit() + + # the update should fail without the force flag + update_invalid_repos_and_check_if_repos_exist( + OperationType.UPDATE, + origin_auth_repo, + client_dir, + FORCED_UPDATE_PATTERN, + True, + ) + + # now call with the force flag + update_and_check_commit_shas( + OperationType.UPDATE, + origin_auth_repo, + client_dir, + force=True, + ) + + assert not target1.something_to_commit() + + setup_manager.add_task( + add_file_to_target_repo_without_committing, kwargs={"target_name": "target1"} + ) + setup_manager.execute_tasks() + + assert target1.something_to_commit() + + # the update should fail without the force flag + update_invalid_repos_and_check_if_repos_exist( + OperationType.UPDATE, + origin_auth_repo, + client_dir, + FORCED_UPDATE_PATTERN, + True, + ) + + update_and_check_commit_shas( + OperationType.UPDATE, + origin_auth_repo, + client_dir, + force=True, + ) + assert not target1.something_to_commit() + + +@pytest.mark.parametrize( + "origin_auth_repo", + [ + { + "targets_config": [{"name": "target1"}, {"name": "target2"}], + }, + ], + indirect=True, +) +def test_update_valid_when_detached_head(origin_auth_repo, client_dir): + clone_repositories( + origin_auth_repo, + client_dir, + ) + client_auth_repo_path = client_dir / origin_auth_repo.name + client_auth_repo = AuthenticationRepository(path=client_auth_repo_path) + + setup_manager = SetupManager(origin_auth_repo) + setup_manager.add_task(update_expiration_dates) + setup_manager.add_task(update_expiration_dates) + setup_manager.add_task(update_expiration_dates) + setup_manager.execute_tasks() + + update_output = update_and_check_commit_shas( + OperationType.UPDATE, + origin_auth_repo, + client_dir, + force=True, + ) + + assert not len(update_output["auth_repos"][client_auth_repo.name]["warnings"]) + + all_commits = client_auth_repo.all_commits_on_branch( + client_auth_repo.default_branch + ) + client_auth_repo.reset_to_commit(all_commits[-2], hard=True) + client_auth_repo.checkout_commit(all_commits[-3]) + assert client_auth_repo.is_detached_head + assert ( + client_auth_repo.top_commit_of_branch(client_auth_repo.default_branch) + == all_commits[-2] + ) + client_auth_repo.set_last_validated_commit(all_commits[-2]) + + update_output = update_and_check_commit_shas( + OperationType.UPDATE, + origin_auth_repo, + client_dir, + force=False, + ) + assert len(update_output["auth_repos"][client_auth_repo.name]["warnings"]) + + assert client_auth_repo.is_detached_head + assert ( + client_auth_repo.top_commit_of_branch(client_auth_repo.default_branch) + == all_commits[-1] + ) + + update_output = update_and_check_commit_shas( + OperationType.UPDATE, + origin_auth_repo, + client_dir, + force=True, + ) + assert not client_auth_repo.is_detached_head + assert ( + client_auth_repo.top_commit_of_branch(client_auth_repo.default_branch) + == all_commits[-1] + ) + assert not len(update_output["auth_repos"][client_auth_repo.name]["warnings"]) + + +@pytest.mark.parametrize( + "origin_auth_repo", + [ + { + "targets_config": [{"name": "target1"}, {"name": "target2"}], + }, + ], + indirect=True, +) +def test_update_valid_when_detached_head_target(origin_auth_repo, client_dir): + # Set up a scenario where repositories + clone_repositories( + origin_auth_repo, + client_dir, + ) + client_auth_repo_path = client_dir / origin_auth_repo.name + client_auth_repo = AuthenticationRepository(path=client_auth_repo_path) + + setup_manager = SetupManager(origin_auth_repo) + setup_manager.add_task(add_valid_target_commits) + setup_manager.add_task(add_valid_target_commits) + setup_manager.add_task(add_valid_target_commits) + setup_manager.execute_tasks() + + update_output = update_and_check_commit_shas( + OperationType.UPDATE, + origin_auth_repo, + client_dir, + force=True, + ) + + assert not len(update_output["auth_repos"][client_auth_repo.name]["warnings"]) + + client_auth_repo_path = GitRepository(client_dir, origin_auth_repo.name).path + client_target_repo_path = client_auth_repo_path.parent / "target1" + client_target_repo = GitRepository(path=client_target_repo_path) + all_commits = client_target_repo.all_commits_on_branch( + client_auth_repo.default_branch + ) + client_target_repo.reset_to_commit(all_commits[-2], hard=True) + client_target_repo.checkout_commit(all_commits[-3]) + assert client_target_repo.is_detached_head + assert ( + client_target_repo.top_commit_of_branch(client_target_repo.default_branch) + == all_commits[-2] + ) + + update_output = update_and_check_commit_shas( + OperationType.UPDATE, + origin_auth_repo, + client_dir, + force=False, + ) + + assert len(update_output["auth_repos"][client_auth_repo.name]["warnings"]) + + assert client_target_repo.is_detached_head + assert ( + client_target_repo.top_commit_of_branch(client_target_repo.default_branch) + == all_commits[-1] + ) + + update_output = update_and_check_commit_shas( + OperationType.UPDATE, + origin_auth_repo, + client_dir, + force=True, + ) + assert not client_target_repo.is_detached_head + assert ( + client_target_repo.top_commit_of_branch(client_target_repo.default_branch) + == all_commits[-1] + ) + assert not len(update_output["auth_repos"][client_auth_repo.name]["warnings"]) + + +@pytest.mark.parametrize( + "origin_auth_repo", + [ + { + "targets_config": [{"name": "target1"}, {"name": "target2"}], + }, + ], + indirect=True, +) +def test_update_with_removed_commits_in_auth_repo(origin_auth_repo, client_dir): + clone_repositories(origin_auth_repo, client_dir) + + setup_manager = SetupManager(origin_auth_repo) + setup_manager.add_task(add_valid_target_commits) + setup_manager.execute_tasks() + + client_auth_repo = AuthenticationRepository(client_dir, origin_auth_repo.name) + remove_commits( + repo_path=str(client_auth_repo.path), + num_commits=1, + ) + + update_invalid_repos_and_check_if_repos_exist( + OperationType.UPDATE, + origin_auth_repo, + client_dir, + BEHIND_LVC_PATTERN, + True, + ) + + update_and_check_commit_shas( + OperationType.UPDATE, + origin_auth_repo, + client_dir, + force=True, + ) + + +@pytest.mark.parametrize( + "origin_auth_repo", + [ + { + "targets_config": [{"name": "target1"}, {"name": "target2"}], + }, + ], + indirect=True, +) +def test_update_with_last_validated_commit_not_in_local_repo( + origin_auth_repo, client_dir +): + clone_repositories(origin_auth_repo, client_dir) + + setup_manager = SetupManager(origin_auth_repo) + setup_manager.add_task(add_valid_target_commits) + setup_manager.execute_tasks() + + origin_top_commit_sha = origin_auth_repo.head_commit_sha() + client_auth_repo = AuthenticationRepository(client_dir, origin_auth_repo.name) + client_auth_repo.set_last_validated_commit(origin_top_commit_sha) + + remove_commits( + repo_path=str(client_auth_repo.path), + num_commits=1, + ) + + update_invalid_repos_and_check_if_repos_exist( + OperationType.UPDATE, + origin_auth_repo, + client_dir, + LVC_NOT_IN_REPO_PATTERN, + True, + ) + + update_and_check_commit_shas( + OperationType.UPDATE, + origin_auth_repo, + client_dir, + force=True, + ) diff --git a/taf/tests/test_updater/test_update/test_update_invalid.py b/taf/tests/test_updater/test_update/test_update_invalid.py index fae84326..f1ef9475 100644 --- a/taf/tests/test_updater/test_update/test_update_invalid.py +++ b/taf/tests/test_updater/test_update/test_update_invalid.py @@ -1,22 +1,15 @@ import pytest from taf.auth_repo import AuthenticationRepository from taf.tests.test_updater.conftest import ( - INVALID_KEYS_PATTERN, + LVC_NOT_IN_REMOTE_PATTERN, TARGET_MISSMATCH_PATTERN, - FORCED_UPDATE_PATTERN, UNCOMMITTED_CHANGES, - COMMIT_NOT_FOUND_PATTERN, SetupManager, - add_file_without_commit, add_unauthenticated_commits_to_all_target_repos, add_valid_target_commits, create_index_lock, - update_expiration_dates, - update_file_without_commit, - update_role_metadata_invalid_signature, ) from taf.tests.test_updater.update_utils import ( - check_if_last_validated_commit_exists, clone_repositories, update_invalid_repos_and_check_if_repos_exist, ) @@ -84,140 +77,6 @@ def test_update_invalid_repo_target_in_indeterminate_state( ) -@pytest.mark.parametrize( - "origin_auth_repo", - [ - { - "targets_config": [{"name": "target1"}, {"name": "target2"}], - }, - ], - indirect=True, -) -def test_dirty_index_auth_repo_update_file(origin_auth_repo, client_dir): - clone_repositories( - origin_auth_repo, - client_dir, - ) - - client_auth_repo_path = client_dir / origin_auth_repo.name - update_file_without_commit(str(client_auth_repo_path), "dirty_file.txt") - - update_invalid_repos_and_check_if_repos_exist( - OperationType.UPDATE, - origin_auth_repo, - client_dir, - FORCED_UPDATE_PATTERN, - True, - ) - - -@pytest.mark.parametrize( - "origin_auth_repo", - [ - { - "targets_config": [{"name": "target1"}, {"name": "target2"}], - }, - ], - indirect=True, -) -def test_dirty_index_auth_repo_add_file(origin_auth_repo, client_dir): - clone_repositories( - origin_auth_repo, - client_dir, - ) - client_auth_repo_path = client_dir / origin_auth_repo.name - add_file_without_commit(str(client_auth_repo_path), "new_file.txt") - - update_invalid_repos_and_check_if_repos_exist( - OperationType.UPDATE, - origin_auth_repo, - client_dir, - FORCED_UPDATE_PATTERN, - True, - ) - - -@pytest.mark.parametrize( - "origin_auth_repo", - [ - { - "targets_config": [{"name": "target1"}, {"name": "target2"}], - }, - ], - indirect=True, -) -def test_dirty_index_target_repo_update_file(origin_auth_repo, client_dir): - clone_repositories( - origin_auth_repo, - client_dir, - ) - - client_target_repo_path = client_dir / origin_auth_repo.name / "namespace/target1" - update_file_without_commit(str(client_target_repo_path), "dirty_file.txt") - - update_invalid_repos_and_check_if_repos_exist( - OperationType.UPDATE, - origin_auth_repo, - client_dir, - FORCED_UPDATE_PATTERN, - True, - ) - - -@pytest.mark.parametrize( - "origin_auth_repo", - [ - { - "targets_config": [{"name": "target1"}, {"name": "target2"}], - }, - ], - indirect=True, -) -def test_dirty_index_target_repo_add_file(origin_auth_repo, client_dir): - clone_repositories( - origin_auth_repo, - client_dir, - ) - - client_target_repo_path = client_dir / origin_auth_repo.name / "namespace/target1" - add_file_without_commit(str(client_target_repo_path), "new_file.txt") - - update_invalid_repos_and_check_if_repos_exist( - OperationType.UPDATE, - origin_auth_repo, - client_dir, - FORCED_UPDATE_PATTERN, - True, - ) - - -@pytest.mark.parametrize( - "origin_auth_repo", - [ - { - "targets_config": [{"name": "target1"}, {"name": "target2"}], - }, - ], - indirect=True, -) -def test_update_invalid_when_repos_not_clean(origin_auth_repo, client_dir): - clone_repositories( - origin_auth_repo, - client_dir, - ) - client_auth_repo_path = client_dir / origin_auth_repo.name - - update_file_without_commit(str(client_auth_repo_path), "dirty_file.txt") - - update_invalid_repos_and_check_if_repos_exist( - OperationType.UPDATE, - origin_auth_repo, - client_dir, - FORCED_UPDATE_PATTERN, - True, - ) - - @pytest.mark.parametrize( "origin_auth_repo", [ @@ -238,7 +97,7 @@ def test_update_with_invalid_last_validated_commit(origin_auth_repo, client_dir) OperationType.UPDATE, origin_auth_repo, client_dir, - COMMIT_NOT_FOUND_PATTERN, + LVC_NOT_IN_REMOTE_PATTERN, True, ) diff --git a/taf/tests/test_updater/test_update/test_update_partial.py b/taf/tests/test_updater/test_update/test_update_partial.py index c3ec9915..a8e6819d 100644 --- a/taf/tests/test_updater/test_update/test_update_partial.py +++ b/taf/tests/test_updater/test_update/test_update_partial.py @@ -1,9 +1,10 @@ import pytest +from taf.taf.auth_repo import AuthenticationRepository from taf.tests.test_updater.conftest import ( SetupManager, add_valid_target_commits, set_head_commit, - update_file_without_commit, + update_target_repo_without_committing, ) from taf.tests.test_updater.update_utils import ( clone_repositories, @@ -27,6 +28,7 @@ def test_update_partial_with_invalid_commits(origin_auth_repo, client_dir): client_dir, ) client_auth_repo_path = client_dir / origin_auth_repo.name + client_auth_repo = AuthenticationRepository(path=client_auth_repo_path) setup_manager = SetupManager(origin_auth_repo) setup_manager.add_task(add_valid_target_commits) @@ -37,9 +39,11 @@ def test_update_partial_with_invalid_commits(origin_auth_repo, client_dir): setup_manager.add_task(add_valid_target_commits) setup_manager.execute_tasks() - update_file_without_commit( - str(client_auth_repo_path / "targets/target1"), "invalid_file.txt" + setup_manager = SetupManager(client_auth_repo) + setup_manager.add_task( + update_target_repo_without_committing, kwargs={"target_name": "target1"} ) + setup_manager.execute_tasks() update_and_check_commit_shas( OperationType.UPDATE, diff --git a/taf/updater/updater_pipeline.py b/taf/updater/updater_pipeline.py index 4ee339a0..50a86c43 100644 --- a/taf/updater/updater_pipeline.py +++ b/taf/updater/updater_pipeline.py @@ -598,7 +598,7 @@ def prepare_for_auth_update_and_check_last_validated_commit(self): self.state.auth_commits_since_last_validated = None # set last validated commit before running the updater # this last validated commit is read from the settings - self._set_last_validated_commit(self.state.validation_auth_repo) + self._set_last_validated_commit() # check if auth path is provided and if that is not the case # check if info.json exists. info.json will be read after validation @@ -610,7 +610,6 @@ def prepare_for_auth_update_and_check_last_validated_commit(self): self.state.validation_auth_repo.default_branch ) ) - self.state.update_handler = None if not self.auth_path: self.state.auth_repo_name = ( _get_repository_name_raise_error_if_not_defined( @@ -630,12 +629,11 @@ def prepare_for_auth_update_and_check_last_validated_commit(self): self.state.last_validated_commit, default_branch, ): - error_msg = ( - f"Last validated commit {self.state.last_validated_commit} is no longer on the {default_branch}" - f"of the remote {self.state.validation_auth_repo.name} repository. This could " + f"Last validated commit {self.state.last_validated_commit} is no longer on {default_branch} " + f"of the remote {self.state.users_auth_repo.name} repository. This could " "either mean that there was an unauthorized push to the remote " - "repository, or that last_validated_commit file was modified." + "repository, or that last_validated_commit file was modified. " ) if self.force: # if last validated commit is not in the remote and run with --force, start the @@ -648,7 +646,9 @@ def prepare_for_auth_update_and_check_last_validated_commit(self): ] = None self.state.last_validated_commit = None else: - raise UpdateFailedError(error_msg) + raise UpdateFailedError( + f"{error_msg}\nRun the updater with the --force flag to run the validation from the first commit" + ) # validate the top commit of the user's auth repo # if it's not in the remote repo -> fail early users_head_sha = self.state.users_auth_repo.top_commit_of_branch( @@ -657,39 +657,65 @@ def prepare_for_auth_update_and_check_last_validated_commit(self): if self.state.last_validated_commit and not _check_if_commit_on_branch( self.state.validation_auth_repo, users_head_sha, default_branch ): - error_msg = ( - f"The newest commit of repository {self.state.users_auth_repo.name} is not longer " + f"The newest commit of repository {self.state.users_auth_repo.name} is no longer " f"on branch {default_branch} of the remote repository. This could " "either mean that there was an unauthorized push to the remote " - "or invalid modification of the local repository." + "or an invalid modification of the local repository." ) # this is always an error, force or no force raise UpdateFailedError(error_msg) + if users_head_sha != self.state.last_validated_commit: - commits_since = self.state.users_auth_repo.all_commits_since_commit( - since_commit=self.state.last_validated_commit, - branch=default_branch, - ) - if users_head_sha not in commits_since: - if self.force: - if _check_if_commit_on_branch( - self.state.users_auth_repo, - self.state.last_validated_commit, - default_branch, - ): - self.state.users_auth_repo.reset_to_commit( - self.state.last_validated_commit + if not _check_if_commit_on_branch( + self.state.users_auth_repo, + self.state.last_validated_commit, + default_branch, + include_remotes=True, + ): + if self.force: + settings.last_validated_commit[ + self.state.validation_auth_repo.name + ] = None + self.state.last_validated_commit = None + taf_logger.warning( + f"{self.state.users_auth_repo.name}: Last validated commit {users_head_sha} is not in repository {self.state.users_auth_repo.name} " + "Running the validation from the first commit." ) - taf_logger.log( - "NOTICE", - f"{self.state.users_auth_repo.name}: Top commit of repository {self.state.users_auth_repo.name} {users_head_sha} is not equal to or newer than the last successful commit. " - "Resetting repository to last validated commit", + else: + raise UpdateFailedError( + f"{self.state.users_auth_repo.name}: Last validated commit {users_head_sha} is not in repository {self.state.users_auth_repo.name} " + "\nRun the updater with the --force flag to run the validation from the first commit" ) else: - raise UpdateFailedError( - f"Top commit of repository {self.state.users_auth_repo.name} {users_head_sha} is not equal to or newer than the last successful commit." + commits_since = ( + self.state.users_auth_repo.all_commits_since_commit( + since_commit=self.state.last_validated_commit, + branch=default_branch, + ) ) + # if the user's head sha is newer than last validated commit + # that can mean that the changes were pulled manually + # validation will start from the last validated commit + # and there is no need to do anything else + # if the user's head commit is not newer or equal to the last validated commit + # that could meant that the user manually removed some commits from the local + # repository + if users_head_sha not in commits_since: + if self.force: + settings.last_validated_commit[ + self.state.validation_auth_repo.name + ] = None + self.state.last_validated_commit = None + taf_logger.warning( + f"{self.state.users_auth_repo.name}: Top commit of repository {self.state.users_auth_repo.name} {users_head_sha} is not equal to or newer than the last successful commit. " + "Running the validation from the first commit." + ) + else: + raise UpdateFailedError( + f"Top commit of repository {self.state.users_auth_repo.name} {users_head_sha} is not equal to or newer than the last successful commit. " + "\nRun the updater with the --force flag to run the validation from the first commit" + ) except Exception as e: self.state.errors.append(e) @@ -738,21 +764,7 @@ def run_tuf_updater(self): self._validate_operation_type() self.state.is_test_repo = self.state.validation_auth_repo.is_test_repo - if self.operation == OperationType.UPDATE: - self._validate_last_validated_commit( - settings.last_validated_commit.get( - self.state.validation_auth_repo.name - ) - ) - # used for testing purposes - if settings.overwrite_last_validated_commit: - self.state.last_validated_commit = settings.last_validated_commit.get( - self.state.validation_auth_repo.name - ) - else: - self.state.last_validated_commit = ( - self.state.users_auth_repo.last_validated_commit - ) + self.state.invalid_auth_commits = [] if error is None: self.state.auth_commits_since_last_validated = list( @@ -1901,15 +1913,24 @@ def _update_tuf_current_revision(git_fetcher, updater, auth_repo_name): ) -def _check_if_commit_on_branch(repo, commit, branch): +def _check_if_commit_on_branch(repo, commit, branch, include_remotes=True): try: repo.commit_exists(commit_sha=commit) except GitError: return False - branches_containing_last_validated_commit = repo.branches_containing_commit(commit) + try: + branches_containing_last_validated_commit = repo.branches_containing_commit( + commit + ) + except GitError: + return False - return branch in branches_containing_last_validated_commit + if branch in branches_containing_last_validated_commit: + return True + if include_remotes: + return f"origin/{branch}" in branches_containing_last_validated_commit + return False def _validate_metadata_on_disk(git_fetcher):