Skip to content

Commit

Permalink
Merge branch 'master' into ndusan/notice-level-update-status
Browse files Browse the repository at this point in the history
  • Loading branch information
renatav committed Sep 28, 2024
2 parents da35421 + 5f61e58 commit 322de17
Show file tree
Hide file tree
Showing 6 changed files with 87 additions and 13 deletions.
7 changes: 4 additions & 3 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -159,25 +159,26 @@ jobs:
- name: Install dependencies
run: |
pip install .[yubikey]
pip install .[executable]
pip install pyinstaller
- name: Build and test standalone executable (Linux)
if: matrix.os == 'ubuntu-latest'
run: |
pyinstaller --onefile --hidden-import=yubikey_manager --name taf-linux -y taf/tools/cli/taf.py
pyinstaller --onefile --hidden-import=yubikey_manager --hidden-import=lxml --collect-submodules taf.tools --name taf-linux -y taf/tools/cli/taf.py
chmod +x dist/taf-linux
./dist/taf-linux --help | grep "TAF Command Line Interface" || { echo "Error: Expected text not found in the executable output"; exit 1; }
- name: Build and test standalone executable (Windows)
if: matrix.os == 'windows-latest'
run: |
pyinstaller --onefile --hidden-import=yubikey_manager --name taf-windows.exe -y taf/tools/cli/taf.py
pyinstaller --onefile --hidden-import=yubikey_manager --hidden-import=lxml --collect-submodules taf.tools --name taf-windows.exe -y taf/tools/cli/taf.py
./dist/taf-windows.exe --help | Select-String "TAF Command Line Interface" -quiet
- name: Build and test standalone executable (macOS)
if: matrix.os == 'macos-latest'
run: |
pyinstaller --onefile --hidden-import=yubikey_manager --name taf-macos -y taf/tools/cli/taf.py
pyinstaller --onefile --hidden-import=yubikey_manager --hidden-import=lxml --collect-submodules taf.tools --name taf-macos -y taf/tools/cli/taf.py
./dist/taf-macos --help | grep "TAF Command Line Interface" || { echo "Error: Expected text not found in the executable output"; exit 1; }
- name: Upload standalone executable (Linux)
Expand Down
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning][semver].
### Added


- Added lxml to taf pyinstaller to execute arbitrary python scripts ([535])
- Added support for execution of executable files within the scripts directories ([529])
- Added yubikey_present parameter to keys description (Can be specified when generating keys) ([508])
- Removed 2048-bit key restriction [494]
Expand Down Expand Up @@ -42,6 +43,8 @@ and this project adheres to [Semantic Versioning][semver].

### Fixed

- Fixes to executing taf handler scripts from a pyinstaller executable ([535])
- Fix `persisent` and `transient` NoneType error when running taf handlers ([535])
- Fix update status when a target repo was updated and the auth repo was not ([532])
- Fix merge-commit which wasn't updating the remote-tracking branch ([532])
- Fix removal of additional local commits ([532])
Expand All @@ -50,6 +53,7 @@ and this project adheres to [Semantic Versioning][semver].
- `check_if_repositories_clean` error now returns a list of repositories which aren't clean, instead of a single repository ([525])

[538]: https://github.com/openlawlibrary/taf/pull/538
[535]: https://github.com/openlawlibrary/taf/pull/535
[532]: https://github.com/openlawlibrary/taf/pull/532
[529]: https://github.com/openlawlibrary/taf/pull/529
[528]: https://github.com/openlawlibrary/taf/pull/528
Expand Down
3 changes: 3 additions & 0 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@
"freezegun==0.3.15",
]

executable_require = ["lxml"]

dev_require = ["bandit>=1.6.0", "black>=19.3b0", "pre-commit>=1.18.3"]

tests_require = [
Expand Down Expand Up @@ -72,6 +74,7 @@
"test": tests_require,
"dev": dev_require,
"yubikey": yubikey_require,
"executable": executable_require,
},
"tests_require": tests_require,
"entry_points": {
Expand Down
3 changes: 2 additions & 1 deletion taf/tools/cli/taf.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import click
from .lazy_group import LazyGroup
from taf.tools.cli.lazy_group import LazyGroup


@click.group(cls=LazyGroup, lazy_subcommands={
Expand All @@ -11,6 +11,7 @@
"metadata": "taf.tools.metadata.attach_to_group",
"roles": "taf.tools.roles.attach_to_group",
"yubikey": "taf.tools.yubikey.attach_to_group",
"scripts": "taf.tools.scripts.attach_to_group",
})
@click.version_option()
def taf():
Expand Down
50 changes: 50 additions & 0 deletions taf/tools/scripts/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
import click

from pathlib import Path
from taf.log import taf_logger

import ast


def extract_global_variables(filename):
"""
Utility function to extract global variables from a Python file.
This is necessary because we want to execute the Python file in a separate process, and we need to pass the global variables to it.
TAF currently uses this when executing lifecycle handler scripts from an executable built from pyinstaller, which uses a frozen sys module.
"""
with open(filename, "r") as f:
tree = ast.parse(f.read(), filename=filename)

global_vars = {}

for node in ast.walk(tree):
try:
if isinstance(node, ast.Assign):
for target in node.targets:
if isinstance(target, ast.Name):
# We only extract simple variable assignments, not function definitions or imports
if isinstance(node.value, (ast.Constant, ast.List, ast.Dict, ast.Tuple, ast.Constant)):
# Convert the AST expression to Python objects
global_vars[target.id] = ast.literal_eval(node.value)
except Exception as e:
taf_logger.debug(f"Error extracting global variables from {filename}: {e}")
pass

global_vars["__file__"] = filename

return global_vars


def execute_command():
@click.command(help="Executes an arbitrary python script")
@click.argument("script_path")
def execute(script_path):
script_path = Path(script_path).resolve()
global_scopes = extract_global_variables(script_path)
with open(script_path, "r") as f:
exec(f.read(), global_scopes) # nosec: B102
return execute


def attach_to_group(group):
group.add_command(execute_command(), name='execute')
33 changes: 24 additions & 9 deletions taf/updater/lifecycle_handlers.py
Original file line number Diff line number Diff line change
Expand Up @@ -190,21 +190,38 @@ def execute_scripts(auth_repo, last_commit, scripts_rel_path, data, scripts_root
script_paths = []

for script_path in sorted(script_paths):
taf_logger.info("Executing script {}", script_path)
taf_logger.log("NOTICE", f"Executing script {script_path}")
json_data = json.dumps(data)
try:
if Path(script_path).suffix == ".py":
output = run(sys.executable, script_path, input=json_data)
if getattr(sys, "frozen", False):
# we are running in a pyinstaller bundle
output = run(
f"{sys.executable}",
"scripts",
"execute",
script_path,
input=json_data,
)
else:
output = run(f"{sys.executable}", script_path, input=json_data)
# assume that all other types of files are non-OS-specific executables of some kind
else:
output = run_subprocess([script_path])
except subprocess.CalledProcessError as e:
taf_logger.error(
"An error occurred while executing {}: {}", script_path, e.output
)
raise ScriptExecutionError(script_path, e.output)
except Exception as e:
if type(e) is subprocess.CalledProcessError:
taf_logger.error(
f"An error occurred while executing {script_path}: {e.output}"
)
raise ScriptExecutionError(script_path, e.output)
else:
taf_logger.error(
f"An error occurred while executing {script_path}: {str(e)}"
)
raise ScriptExecutionError(script_path, str(e))
if type(output) is bytes:
output = output.decode()
transient_data = persistent_data = {}
if output is not None and output != "":
# if the script contains print statements other than the final
# print which outputs transient and persistent data
Expand All @@ -215,8 +232,6 @@ def execute_scripts(auth_repo, last_commit, scripts_rel_path, data, scripts_root
persistent_data = json_data.get("persistent")
if transient_data is not None or persistent_data is not None:
break
else:
transient_data = persistent_data = {}
taf_logger.debug("Persistent data: {}", persistent_data)
taf_logger.debug("Transient data: {}", transient_data)
# overwrite current persistent and transient data
Expand Down

0 comments on commit 322de17

Please sign in to comment.