Merge branch 'feature/handlers' into handler-invocation-support

openlawlibrary · Sep 18, 2024 · a5d8d91 · a5d8d91
2 parents fc31586 + a163ea2
commit a5d8d91
Show file tree

Hide file tree

Showing 22 changed files with 360 additions and 86 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -9,9 +9,12 @@ and this project adheres to [Semantic Versioning][semver].
 
 ### Added
 
+- Added yubikey_present parameter to keys description (Can be specified when generating keys) ([508])
+- Removed 2048-bit key restriction [494]
 - Allow for the displaying of varied levels of log and debug information based on the verbosity level ([493])
 - Added new tests to test out of sync repositories and manual updates ([488], [504])
-- Added lazy loading to CLI ([481])
+- Update when auth repo's top commit is behind last validated commit [490]
+- Added lazy loading to CLI [481]
 - Testing repositories with dependencies ([479], [487])
 - Hid plaintext when users are prompted to insert YubiKey and press ENTER ([473])
 - Added functionality for parallel execution of child repo during clone and update for performance enhancement ([472])
@@ -41,12 +44,18 @@ and this project adheres to [Semantic Versioning][semver].
 
 ### Fixed
 
+- Fix setup role when specifying public keys in keys-description ([511])
+
+[511]: https://github.com/openlawlibrary/taf/pull/511
+[508]: https://github.com/openlawlibrary/taf/pull/508
 [504]: https://github.com/openlawlibrary/taf/pull/504
+[494]: https://github.com/openlawlibrary/taf/pull/494
 [493]: https://github.com/openlawlibrary/taf/pull/493
+[490]: https://github.com/openlawlibrary/taf/pull/490
 [489]: https://github.com/openlawlibrary/taf/pull/489
 [488]: https://github.com/openlawlibrary/taf/pull/488
 [487]: https://github.com/openlawlibrary/taf/pull/487
-[487]: https://github.com/openlawlibrary/taf/pull/485
+[485]: https://github.com/openlawlibrary/taf/pull/485
 [481]: https://github.com/openlawlibrary/taf/pull/481
 [479]: https://github.com/openlawlibrary/taf/pull/479
 [473]: https://github.com/openlawlibrary/taf/pull/473
@@ -86,6 +95,23 @@ and this project adheres to [Semantic Versioning][semver].
 [391]: https://github.com/openlawlibrary/taf/pull/391
 [389]: https://github.com/openlawlibrary/taf/pull/389
 
+## [0.30.2] - 08/20/2024
+
+### Added
+
+- New flag --no-deps allowing users to only update the current repository and not update dependent repositories from dependencies.json ([455])
+- New flag --no-targets allowing users to skip target repository validation when validating the authentication repo ([455])
+- New flag --no-upstream allowing users to skip upstream comparisons ([455])
+
+- Addition of logic to tuples (steps) and the run function in updater_pipeline.py to determine which steps, if any, will be skipped based on the usage of 
+  the --no-targets flag ([455])
+
+### Changed
+
+### Fixed
+
+[463]: https://github.com/openlawlibrary/taf/pull/463
+[455]: https://github.com/openlawlibrary/taf/pull/455
 
 ## [0.30.1] - 07/23/2024
 
@@ -1202,7 +1228,8 @@ and this project adheres to [Semantic Versioning][semver].
 
 [keepachangelog]: https://keepachangelog.com/en/1.0.0/
 [semver]: https://semver.org/spec/v2.0.0.html
-[unreleased]: https://github.com/openlawlibrary/taf/compare/v0.30.1...HEAD
+[unreleased]: https://github.com/openlawlibrary/taf/compare/v0.30.2...HEAD
+[0.30.2]: https://github.com/openlawlibrary/taf/compare/v0.30.1...v0.30.2
 [0.30.1]: https://github.com/openlawlibrary/taf/compare/v0.30.0...v0.30.1
 [0.30.0]: https://github.com/openlawlibrary/taf/compare/v0.29.1...v0.30.0
 [0.29.1]: https://github.com/openlawlibrary/taf/compare/v0.29.0...v0.29.1

diff --git a/docs/updater/states.md b/docs/updater/states.md
@@ -0,0 +1,81 @@
+# Different states in TAF could be:
+
+```
+args:
+    - `--force` - force all repositories into consistent state at LVC and then run updater; may lose data. provide info on all changes made to repositories in order get to consistent state (use reflog if needed)
+
+
+LVC = Last Validated Commit
+
+axis:
+    - working directory
+        - clean  
+            - happy path
+        - ~dirty~
+            - DEFAULT: FAIL
+            - --force: git clean -fd; git reset HEAD --hard; proceed with update
+    - branch
+        - correct
+            - happy path
+        - incorrect branch or detached HEAD?
+            - DEFAULT: stay on the incorrect branch; update correct branch; provide warning that not on the correct branch
+            - --force: switch to LVC branch; proceed with update
+    - commit
+        - at last validated commit
+            - happy path
+        - validatable (in authentication chain, even if it is in "future" relative to last validated commit; this could happen if you manually pull or reset --hard)
+            - DEFAULT: update to LVC; proceed with update
+        - not validatable (not in validation chain)
+            - DEFAULT: FAIL
+            - --force: force branch to LVC; proceed with update
+        - trailing unvalidated commits (in law-xml-codified but not law) - we call them unauthenticated in updater
+            - DEFAULT: FAIL
+            - --force: reset LVC --hard; proceed with update
+    - local repositories (all commits are consistent at repository level)
+        - doesn't exist, no commit
+            - DEFAULT: create repo; ff to LVC
+    - local vs remote
+        - remote is consistent with local - no trailing unauthenticated commits
+            - happy path
+        - remote is consistent with local - trailing unauthenticated commits on remote 
+            - DEFAULT: update to last validatable commit on remote; then fail for any unauthenticate commits on repos that don't allow; notify of commits on repos that do allow
+        - remote is consistent with local - trailing unauthenticated commits on local
+            - DEFAULT: update to the last authenticatable commit on remote; FAIL if repo does not allow trailing commits or do nothing if repo allows trailing commits
+            - --force: remove trailing commits
+        - remote is consistent with local - trailing unauthenticated commits conflict on repo that allows trailing commits
+            - DEFAULT: update to the last authenticatable commit on remote; FAIL
+            - --force: update to the last authenticatable commit on remote; remove local commits
+        - remote is inconsistent with local (i.e. both remote and local are valid but diverge)
+            - DEFAULT: update to the last authenticable commit on remote; FAIL
+        - remote state is not authenticatable (metadata issues, versions, keys, signing, etc.)
+            - DEFAULT: Update to last authenticatable remote commit; then FAIL
+
+States to specifically address:
+    - html repo has commit with no corresponding law commit (remote is not valid so: Update to last valid/authenticatable remote commit; then FAIL)
+    - local repository has valid commits that aren't on remote (remote is not valid so: Update to last valid/authenticatable remote commit; then FAIL)
+
+Update algorithm:
+
+- prep for update
+    - this is where we address:
+        - working directory axis
+        - branch axis
+        - commit axis (excluding commits past LVC)
+        - local repositories axis
+    - Get all local repositories consistent at LVC (EXCEPTION: allow commits past LVC)
+        - --force: do everything you possibly can to get them to that state, if can't then fail
+        - DEFAULT: can use:
+            - fast forward
+            - git clone for repos that have been removed
+            - allowed to have commits past LVC - they will be validated during update process or FAIL at that point
+- update
+    - this is where we address:
+        - local vs remote axis
+        - commits past LVC (from commit axis)
+ ```
+
+
+
+
+
+
diff --git a/docs/updater/update_process.md b/docs/updater/update_process.md
@@ -57,6 +57,7 @@ specified (similarly to how `git pull` works)
 - `--format-output`: Return formatted output including information on whether the build was successful and an error message if it was raised.
 - `--exclude-target`: Globs defining which target repositories should be ignored during the update. Accepts multiple values.
 - `--strict`: Enable/disable strict mode. In strict mode, an error is returned if warnings are raised. Default is disabled.
+- `--force`: Flag used to run a forced update.
 
 ### Determining filesystem paths of repositories
 
@@ -158,3 +159,12 @@ if __name__ == '__main__':
     state = do_something(data)
     send_state(state)
 ```
+
+
+# Forced Updates
+
+A forced update overrides the normal update process to address situations where the authentication repository or its target repositories have encountered inconsistencies that prevent a standard update. During a forced update, the updater bypasses certain checks that would normally stop the process, such as the presence of invalid or uncommitted changes. The forced update ensures that the repository is realigned with a valid state.
+
+By using the --force flag, the updater will reset the state of the repository, discarding any uncommitted changes, and align the repository with the last validated commit or a validated state in the remote repository. This process includes removing certain unvalidated commits or local changes that could conflict with the validated state. If the target repositories have diverged or contain unaligned commits, the updater will reset them to the validated state, potentially discarding local changes. The last validated commit is not automatically set to the latest remote commit unless that commit has been properly validated; instead, the updater ensures the repository is consistent with the authenticated and validated state.
+
+However, if the remote repository itself is in an inconsistent state—such as having missing critical metadata, or if the commit history has been rewritten, the forced update may still fail. For example, if a file is updated but not committed in the authentication repository, this creates a "dirty index" which makes the updater fail. 
diff --git a/docs/updater/updater.md b/docs/updater/updater.md
@@ -25,3 +25,7 @@ For the above listed reasons, we cannot simply use TUF's updater like described
 - New metadata files (not counting delegated roles) are acquired by calling TUF's `refresh`. Metadata files belonging to the delegated roles are updated by `get_current_targets`. Information about a target file is obtained using `get_one_valid_targetinfo`
 
 - Since TUF does not provide a way to gradually update metadata and target files (except for `root.json`), that has to be handled by TAF. So, TAF's updater calls TUF's updater multiple times. Our handler, `GitUpdateHandler` finds all commits that were pushed after the last time the client pulled the changes. The basic idea is to traverse trough these commits and update metadata files and targets for each of them, thus validating all revisions.
+
+- Validation and syncing are critical for ensuring the updater operates correctly. The updater automatically handles these tasks by checking for any discrepancies between the local repositories and their respective last validated states. If a repository is in a invalid state and a forced update is run, the updater identifies and removes any unvalidated or unauthorized commits, discarding uncommitted changes, and ensuring that all repositories are aligned with their last validated commit. Othervise, the updater will throw an error and fail. Examples of when validation and syncing are necessary include when the client’s authentication repository is out of sync with the remote repository, when target repositories have diverged, or when there are mixed states due to manual alterations. 
+
+- In the updater, validating the last validated commit ensures that the reference commit for updates remains valid and consistent with the repository's current state. The process begins by identifying the main branch and verifying the latest commit on that branch. If the last validated commit is already set but is not present in the local repository, the updater checks if this commit exists in the remote repository without raising an error. The updater will not automatically fetch or set the last validated commit to any unvalidated commits. Instead, it ensures that any updates are based on a consistent and authenticated state, thereby preventing issues that could arise from unvalidated commits.
diff --git a/setup.py b/setup.py
@@ -3,7 +3,7 @@
 import sys
 
 PACKAGE_NAME = "taf"
-VERSION = "0.30.1"
+VERSION = "0.30.2"
 AUTHOR = "Open Law Library"
 AUTHOR_EMAIL = "info@openlawlib.org"
 DESCRIPTION = "Implementation of archival authentication"

diff --git a/taf/api/roles.py b/taf/api/roles.py
@@ -498,12 +498,15 @@ def _enter_role_info(
         while click.confirm(
             f"Add {'another' if len(delegated_roles) else 'a'} delegated targets role of role {role}?"
         ):
-            role_name = _read_val(str, "role name", True)
+            role_name = _read_val(str, "role name", "role_name", True)
             delegated_paths: List[str] = []
             while not len(delegated_paths) or click.confirm("Enter another path?"):
                 delegated_paths.append(
                     _read_val(
-                        str, f"path or glob pattern delegated to {role_name}", True
+                        str,
+                        f"path or glob pattern delegated to {role_name}",
+                        "delegated_paths",
+                        True,
                     )
                 )
             delegated_roles[role_name]["paths"] = delegated_paths

diff --git a/taf/api/targets.py b/taf/api/targets.py
@@ -368,7 +368,7 @@ def register_target_files(
         prompt_for_keys=prompt_for_keys,
     )
 
-    if write:
+    if updated and write:
         taf_repo.writeall()
         if commit:
             auth_repo = AuthenticationRepository(path=taf_repo.path)

diff --git a/taf/api/utils/_roles.py b/taf/api/utils/_roles.py
@@ -1,3 +1,4 @@
+import tuf
 from logging import DEBUG, INFO
 from typing import Dict, List, Optional, Union
 from functools import partial
@@ -95,7 +96,7 @@ def get_roles_and_paths_of_key(
 @log_on_end(DEBUG, "Finished setting up role {role.name:s}", logger=taf_logger)
 def setup_role(
     role: Role,
-    repository: Repository,
+    repository: TUFRepository,
     verification_keys: Dict,
     signing_keys: Optional[Dict] = None,
     parent: Optional[Targets] = None,
@@ -125,6 +126,14 @@ def setup_role(
                 role_obj.add_external_signature_provider(
                     key, partial(yubikey_signature_provider, key_name, key["keyid"])
                 )
+        # Even though we add all verification keys (public keys directly specified in the keys-description)
+        # and those loaded from YubiKeys, only those directly specified in keys-description are registered
+        # as previous_keys
+        # this means that TUF expects at least one of those signing keys to be present
+        # we are setting up this role, so there should be no previous keys
+        tuf.roledb._roledb_dict[repository._repository_name][role.name][
+            "previous_keyids"
+        ] = []
 
 
 def _role_obj(

diff --git a/taf/api/yubikey.py b/taf/api/yubikey.py
@@ -121,7 +121,9 @@ def get_yk_roles(path: str) -> Dict:
     on_exceptions=TAFError,
     reraise=True,
 )
-def setup_signing_yubikey(certs_dir: Optional[str] = None) -> None:
+def setup_signing_yubikey(
+    certs_dir: Optional[str] = None, key_size: int = 2048
+) -> None:
     """
     Delete everything from the inserted YubiKey, generate a new key and copy it to the YubiKey.
     Optionally export and save the certificate to a file.
@@ -146,7 +148,7 @@ def setup_signing_yubikey(certs_dir: Optional[str] = None) -> None:
         pin_repeat=True,
         prompt_message="Please insert the new Yubikey and press ENTER",
     )
-    key = yk.setup_new_yubikey(serial_num)
+    key = yk.setup_new_yubikey(serial_num, key_size=key_size)
     yk.export_yk_certificate(certs_dir, key)
 
 

diff --git a/taf/keys.py b/taf/keys.py
@@ -330,6 +330,7 @@ def setup_roles_keys(
     yubikeys: Optional[Dict] = None,
     users_yubikeys_details: Optional[Dict[str, UserKeyData]] = None,
     skip_prompt: Optional[bool] = False,
+    key_size: int = 2048,
 ):
 
     if role.name is None:
@@ -347,7 +348,7 @@ def setup_roles_keys(
 
     if is_yubikey:
         yubikey_keys = _setup_yubikey_roles_keys(
-            yubikey_ids, users_yubikeys_details, yubikeys, role, certs_dir
+            yubikey_ids, users_yubikeys_details, yubikeys, role, certs_dir, key_size
         )
     else:
         if keystore is None:
@@ -374,19 +375,24 @@ def setup_roles_keys(
 
 
 def _setup_yubikey_roles_keys(
-    yubikey_ids, users_yubikeys_details, yubikeys, role, certs_dir
+    yubikey_ids, users_yubikeys_details, yubikeys, role, certs_dir, key_size
 ):
     loaded_keys_num = 0
     yk_with_public_key = {}
     yubikey_keys = []
+
     for key_id in yubikey_ids:
+        # Check the present value from the yubikeys dictionary
+        if not users_yubikeys_details[key_id].present:
+            continue
+
         public_key_text = None
         if key_id in users_yubikeys_details:
             public_key_text = users_yubikeys_details[key_id].public
         if public_key_text:
             scheme = users_yubikeys_details[key_id].scheme
             public_key = keys.import_rsakey_from_public_pem(public_key_text, scheme)
-            # check if signing key already loaded too
+            # Check if the signing key is already loaded
             if not yk.get_key_serial_by_id(key_id):
                 yk_with_public_key[key_id] = public_key
             else:
@@ -397,10 +403,17 @@ def _setup_yubikey_roles_keys(
                 key_scheme = users_yubikeys_details[key_id].scheme
             key_scheme = key_scheme or role.scheme
             public_key = _setup_yubikey(
-                yubikeys, role.name, key_id, yubikey_keys, key_scheme, certs_dir
+                yubikeys,
+                role.name,
+                key_id,
+                yubikey_keys,
+                key_scheme,
+                certs_dir,
+                key_size,
             )
             loaded_keys_num += 1
         yubikey_keys.append(public_key)
+
     if loaded_keys_num < role.threshold:
         print(f"Threshold of role {role.name} is {role.threshold}")
         while loaded_keys_num < role.threshold:
@@ -414,11 +427,12 @@ def _setup_yubikey_roles_keys(
                     break
             if loaded_keys_num < role.threshold:
                 if not click.confirm(
-                    f"Threshold of sining keys of role {role.name} not reached. Continue?"
+                    f"Threshold of signing keys of role {role.name} not reached. Continue?"
                 ):
                     raise SigningError("Not enough signing keys")
                 for key_id in loaded_keys:
                     yk_with_public_key.pop(key_id)
+
     return yubikey_keys
 
 
@@ -526,6 +540,7 @@ def _setup_yubikey(
     loaded_keys: List[str],
     scheme: Optional[str] = DEFAULT_RSA_SIGNATURE_SCHEME,
     certs_dir: Optional[Union[Path, str]] = None,
+    key_size: int = 2048,
 ) -> Dict:
     print(f"Registering keys for {key_name}")
     while True:
@@ -551,7 +566,7 @@ def _setup_yubikey(
             print("Key already loaded. Please insert a different YubiKey")
         else:
             if not use_existing:
-                key = yk.setup_new_yubikey(serial_num, scheme)
+                key = yk.setup_new_yubikey(serial_num, scheme, key_size=key_size)
 
             if certs_dir is not None:
                 yk.export_yk_certificate(certs_dir, key)

diff --git a/taf/models/types.py b/taf/models/types.py
@@ -21,6 +21,7 @@ class UserKeyData:
         validator=optional_type_validator(str),
         default=DEFAULT_ROLE_SETUP_PARAMS["scheme"],
     )
+    present: bool = attrs.field(default=True)
 
 
 @attrs.define

diff --git a/taf/tests/test_api/test_dependencies.py b/taf/tests/test_api/test_dependencies.py
@@ -67,7 +67,7 @@ def test_add_dependency_when_on_filesystem_invalid_commit(
             dependency_name=child_repository.name,
             keystore=api_keystore,
             branch_name="main",
-            out_of_band_commit="12345678910",
+            out_of_band_commit="66d7f48e972f9fa25196523f469227dfcd85c994",
             no_prompt=True,
             push=False,
         )