-
Notifications
You must be signed in to change notification settings - Fork 661
Github Actions CI implementation
rr needs self-hosted Github Actions runners because we need cloud VMs that can use the hardware PMU (which the Github free Azure-based runners do not support) and root privileges to configure the kernel appropriately. To provide these runners at low cost, our Actions workflow creates AWS spot instances when the workflow begins, runs the actual build and tests on those spot instances, and then shuts down the spot instances.
This is hard to implement safely because creating AWS instances requires AWS credentials, but we want rr PRs containing untrusted code to run the workflows that create the instances, and those PRs can steal any credentials accessible to those workflows. We also don't want to give those PRs any kind of write access to the rr repository. All the existing approaches to this problem that I was able to find are either complex (depend on a lot of third-party code) or have poor security properties (ruling out untrusted PRs) or both.
I settled on a fairly simple solution. We have an AWS Lambda function that exposes a public, unauthenticated HTTP REST API that creates AWS spot instances and registers them as rr runners (and a corresponding API that shuts down those instances). A snapshot of the source code for this Lambda is attached below. The Lambda has an AWS role (not attached here) that lets it manipulate the spot instances. It also contains a Github fine-grained personal access token that provides read-write access to the organization's self-hosted runner registrations and nothing else. Because the API is public and unauthenticated, our workflows do not need or have access to any secrets. A malicious actor can trigger our API endpoint directly, but all they can do is spawn idle rr test runners, which achieves nothing useful for the miscreant.
To make this approach robust against failures to clean up, each spot instance is configured to terminate itself after one hour. Also, every time we unregister any self-hosted runner, we also unregister all offline runners (i.e. runners that have terminated for any reason apart from the usual cleanup). Each instance is labeled with a unique ID for the workflow that created it, and that workflow's job only runs on that runner, so as long as no run takes more than an hour things will work OK. At time of writing our workflows complete in under ten minutes; instance startup and shutdown overhead is only about a minute.
import base64
import boto3
import botocore
import json
import math
import re
import time
import urllib3
github_token = "github_pat_..."
# If you change the instance types, make sure to update the allowed instance types in the ManageRrRunners policy.
instance_types = {"x86_64": "c5.9xlarge", "arm64": "m7g.8xlarge"}
security_group_id = "sg-..."
org = "rr-debugger"
actions_runner_arch = {"x86_64": "x64", "arm64": "arm64"}
poll_interval_seconds = 5
runner_registration_deadline_seconds = 60
vm_instance_lifetime_minutes = 30
http = urllib3.PoolManager()
ec2_client = boto3.client("ec2")
def validate_response(response):
if response.status < 200 or response.status > 299:
raise ValueError(f"API call failed with {response.status}: {response.read()}")
def runner_latest_version():
url = "https://github.com/actions/runner/releases/latest"
response = http.request("HEAD", url)
m = re.compile('https://github.com/actions/runner/releases/tag/v([0-9.]+)').match(response.geturl())
if not m:
raise ValueError(f"Invalid response URL {response.geturl()}")
return m.group(1)
def fetch_github_token(kind):
url = f"https://api.github.com/orgs/{org}/actions/runners/{kind}"
headers = {
"Accept": "application/vnd.github+json",
"Authorization": f"Bearer {github_token}",
"X-GitHub-Api-Version": "2022-11-28",
}
response = http.request("POST", url, headers=headers)
validate_response(response)
data = json.loads(response.data)
return data["token"]
def remove_runner(runner):
runner_id = runner["id"]
print(f"Removing runner", runner)
url = f"https://api.github.com/orgs/{org}/actions/runners/{runner_id}"
headers = {
"Accept": "application/vnd.github+json",
"Authorization": f"Bearer {github_token}",
"X-GitHub-Api-Version": "2022-11-28",
}
http.request("DELETE", url, headers=headers)
# Some other instance of this lambda may clean up runners concurrently, so
# that request can fail, so don't validate it.
def list_runners():
url = f"https://api.github.com/orgs/{org}/actions/runners"
headers = {
"Accept": "application/vnd.github+json",
"Authorization": f"Bearer {github_token}",
"X-GitHub-Api-Version": "2022-11-28",
}
response = http.request("GET", url, headers=headers)
validate_response(response)
data = json.loads(response.data)
return data["runners"]
def remove_runners(label):
for runner in list_runners():
# Any offline runner should be cleaned up too
if (runner["status"] == "offline" or
any(rlabel["name"] == label for rlabel in runner["labels"])):
remove_runner(runner)
def vm_user_data(registration_token, architecture, label, runner_version):
runner_arch = actions_runner_arch[architecture]
download_url = f"https://github.com/actions/runner/releases/download/v{runner_version}/actions-runner-linux-{runner_arch}-{runner_version}.tar.gz"
return f"""#!/bin/bash
shutdown +{vm_instance_lifetime_minutes}
mount -t tmpfs -o size=32G tmpfs /tmp
sudo -u ubuntu --login <<EOF
mkdir /tmp/actions-runner && cd /tmp/actions-runner
curl -o actions-runner.tar.gz -L {download_url}
echo \"Github Runner Installed\"
tar xzf ./actions-runner.tar.gz
echo \"Github Runner Installer Extracted\"
yes '' | ./config.sh --url https://github.com/rr-debugger --token {registration_token} --labels {label},arch_{architecture} --disableupdate
yes '' | ./run.sh
EOF
"""
def create_one(label, architecture, image_id, registration_token, runner_version):
user_data = vm_user_data(registration_token, architecture, label, runner_version)
encoded_user_data = base64.b64encode(user_data.encode('utf-8')).decode('utf-8')
run_instance_params = {
"ImageId": image_id,
"InstanceType": instance_types[architecture],
"UserData": encoded_user_data,
"MinCount": 1,
"MaxCount": 1,
"SecurityGroupIds": [security_group_id],
"InstanceMarketOptions": {"MarketType": "spot"},
"InstanceInitiatedShutdownBehavior": "terminate",
"KeyName": "rr-testing",
"TagSpecifications": [{
"ResourceType": "instance",
"Tags": [{
"Key": "Label",
"Value": label,
}, {
"Key": "Name",
"Value": f"rr test runner {architecture}",
}],
}],
}
try:
print("Launching spot instance with image id", image_id, "for", architecture)
instance_id = ec2_client.run_instances(**run_instance_params)["Instances"][0]["InstanceId"]
except botocore.exceptions.ClientError as error:
if error.response['Error']['Code'] == 'InsufficientInstanceCapacity':
print("Launching non-spot instance with image id", image_id, "for", architecture)
del run_instance_params['InstanceMarketOptions']
instance_id = ec2_client.run_instances(**run_instance_params)["Instances"][0]["InstanceId"]
else:
raise
print("Launched with instance id", instance_id)
return instance_id
def image_id_for_architecture(images, architecture):
for image in images["Images"]:
if image["Architecture"] == architecture:
print("Selecting image", image)
return image["ImageId"]
raise ValueError(f"No image found for architecture {architecture}")
def create(label, architectures):
image_filters = [
{"Name": "architecture", "Values": architectures},
{"Name": "name", "Values": ["ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-*"]},
]
images = ec2_client.describe_images(Owners=["099720109477"], Filters=image_filters)
runner_version = runner_latest_version()
print("Runner version", runner_version)
registration_token = fetch_github_token("registration-token")
instance_ids = []
for architecture in architectures:
image_id = image_id_for_architecture(images, architecture)
instance_ids.append(create_one(label, architecture, image_id, registration_token, runner_version))
print("Waiting for", instance_ids)
ec2_client.get_waiter("instance_running").wait(InstanceIds=instance_ids)
for i in range(math.ceil(runner_registration_deadline_seconds/poll_interval_seconds)):
print("Polling runners")
runners = list_runners()
our_runner_count = 0
for runner in list_runners():
if any(rlabel["name"] == label for rlabel in runner["labels"]):
print("Found runner", runner)
if runner["status"] == "online":
our_runner_count += 1
if our_runner_count == len(architectures):
return {
"statusCode": 200,
"body": "",
}
time.sleep(poll_interval_seconds)
raise ValueError("Timeout waiting for runners to register")
def destroy(label):
filters = [{
"Name": "tag:Label",
"Values": [label],
}]
instance_ids = []
reservations = ec2_client.describe_instances(Filters=filters)['Reservations']
for reservation in reservations:
for instance in reservation['Instances']:
instance_ids.append(instance['InstanceId'])
print("Destroying", instance_ids)
if instance_ids:
try:
# This might fail if the instance(s) shut themselves down
# concurrently
ec2_client.terminate_instances(InstanceIds=instance_ids)
except botocore.exceptions.ClientError as error:
pass
remove_runners(label)
return {
"statusCode": 200,
"body": "",
}
def validate_label(label):
if re.compile(r"rr_runner_([a-zA-Z_0-9]+)").match(label):
return label
raise ValueError("Invalid label")
# A label is just a unique string generated for a specific Actions workflow.
# Each EC2 instance is tagged with its label and architecture.
# Each GHA runner is also labeled with the label and architecture.
def lambda_handler(event, context):
print("Request body:", event["body"])
payload = json.loads(event["body"])
operation = payload["operation"]
if operation == "create":
# Create VMs for the given architectures with the given label.
# Returns when all VMs have registered their Actions runners.
return create(validate_label(payload['label']), payload["architectures"])
elif operation == "destroy":
# Destroy all VMs for the given label.
return destroy(validate_label(payload['label']))
else:
raise ValueError("Invalid operation")