Cleaning up leftover runners for org/repo

.github/workflows/integration-tests.yml

@@ -72,6 +72,7 @@ jobs:
           ORG_NAME: gsamfira
           REPO_NAME: garm-testing
           CREDENTIALS_NAME: test-garm-creds
+          GH_TOKEN: ${{ secrets.GH_OAUTH_TOKEN }}
 
       - name: Show GARM logs
         if: always()

test/integration/e2e.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"log"
@@ -24,6 +25,8 @@ import (
 	"github.com/cloudbase/garm/params"
 	"github.com/go-openapi/runtime"
 	openapiRuntimeClient "github.com/go-openapi/runtime/client"
+	"github.com/google/go-github/v53/github"
+	"golang.org/x/oauth2"
 )
 
 var (
@@ -51,6 +54,7 @@ var (
 	email    = os.Getenv("GARM_EMAIL")
 	name     = os.Getenv("GARM_NAME")
 	baseURL  = os.Getenv("GARM_BASE_URL")
+	ghtoken  = os.Getenv("GH_TOKEN")
 
 	poolID string
 )
@@ -60,10 +64,17 @@ var (
 // ///////////////////
 func handleError(err error) {
 	if err != nil {
-		log.Fatalf("error encountered: %v", err)
+		panic(fmt.Sprintf("error encountered: %s", err))
 	}
 }
 
+func getGithubClient() *github.Client {
+	ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: ghtoken})
+	tc := oauth2.NewClient(context.Background(), ts)
+
+	return github.NewClient(tc)
+}
+
 func printResponse(resp interface{}) {
 	b, err := json.MarshalIndent(resp, "", "  ")
 	handleError(err)
@@ -481,6 +492,75 @@ func FirstRun() {
 	printResponse(user)
 }
 
+// ///////////
+// Cleanup //
+// ///////////
+func GracefulCleanup() {
+	DisableRepoPool()
+	DisableOrgPool()
+
+	DeleteInstance(repoInstanceName)
+	DeleteInstance(orgInstanceName)
+
+	WaitRepoPoolNoInstances()
+	WaitOrgPoolNoInstances()
+
+	DeleteRepoPool()
+	DeleteOrgPool()
+	DeletePool()
+
+	DeleteRepo()
+	DeleteOrg()
+}
+
+func GhOrgRunnersCleanup() {
+	if orgPoolID == "" {
+		log.Println(">>> No organization pool ID provided, skipping organization runners cleanup")
+		return
+	}
+
+	log.Println(">>> Org Github runners cleanup")
+	client := getGithubClient()
+	ghOrgRunners, _, err := client.Actions.ListOrganizationRunners(context.Background(), orgName, nil)
+	handleError(err)
+
+	// Remove organization runners
+	poolLabel := fmt.Sprintf("runner-pool-id:%s", orgPoolID)
+	for _, orgRunner := range ghOrgRunners.Runners {
+		for _, label := range orgRunner.Labels {
+			if label.GetName() == poolLabel {
+				log.Printf(">>> Removing organization runner %s", orgRunner.GetName())
+				_, err := client.Actions.RemoveOrganizationRunner(context.Background(), orgName, orgRunner.GetID())
+				handleError(err)
+			}
+		}
+	}
+}
+
+func GhRepoRunnersCleanup() {
+	if repoPoolID == "" {
+		log.Println(">>> No repository pool ID provided, skipping repository runners cleanup")
+		return
+	}
+
+	log.Println(">>> Repo Github runners cleanup")
+	client := getGithubClient()
+	ghRepoRunners, _, err := client.Actions.ListRunners(context.Background(), orgName, repoName, nil)
+	handleError(err)
+
+	// Remove repository runners
+	poolLabel := fmt.Sprintf("runner-pool-id:%s", repoPoolID)
+	for _, repoRunner := range ghRepoRunners.Runners {
+		for _, label := range repoRunner.Labels {
+			if label.GetName() == poolLabel {
+				log.Printf(">>> Removing repository runner %s", repoRunner.GetName())
+				_, err := client.Actions.RemoveRunner(context.Background(), orgName, repoName, repoRunner.GetID())
+				handleError(err)
+			}
+		}
+	}
+}
+
 // ////////////////////////////
 // Credentials and Providers //
 // ////////////////////////////
@@ -620,13 +700,31 @@ func UpdateRepoPool() {
 }
 
 func DisableRepoPool() {
+	if repoID == "" {
+		log.Println(">>> No repo ID provided, skipping disable repo pool")
+		return
+	}
+	if repoPoolID == "" {
+		log.Println(">>> No repo pool ID provided, skipping disable repo pool")
+		return
+	}
+
 	enabled := false
 	_, err := updateRepoPool(cli, authToken, repoID, repoPoolID, params.UpdatePoolParams{Enabled: &enabled})
 	handleError(err)
 	log.Printf("repo pool %s disabled", repoPoolID)
 }
 
 func WaitRepoPoolNoInstances() {
+	if repoID == "" {
+		log.Println(">>> No repo ID provided, skipping repo pool wait no instances")
+		return
+	}
+	if repoPoolID == "" {
+		log.Println(">>> No repo pool ID provided, skipping repo pool wait no instances")
+		return
+	}
+
 	for {
 		log.Println(">>> Wait until repo pool has no instances")
 		pool, err := getRepoPool(cli, authToken, repoID, repoPoolID)
@@ -665,7 +763,7 @@ func WaitRepoInstance(timeout time.Duration) {
 	handleError(err)
 	printResponse(instanceDetails)
 
-	log.Fatalf("Failed to wait for repo instance to be ready")
+	panic("Failed to wait for repo instance to be ready")
 }
 
 func ListRepoInstances() {
@@ -676,13 +774,27 @@ func ListRepoInstances() {
 }
 
 func DeleteRepo() {
+	if repoID == "" {
+		log.Println(">>> No repo ID provided, skipping delete repo")
+		return
+	}
+
 	log.Println(">>> Delete repo")
 	err := deleteRepo(cli, authToken, repoID)
 	handleError(err)
 	log.Printf("repo %s deleted", repoID)
 }
 
 func DeleteRepoPool() {
+	if repoID == "" {
+		log.Println(">>> No repo ID provided, skipping delete repo pool")
+		return
+	}
+	if repoPoolID == "" {
+		log.Println(">>> No repo pool ID provided, skipping delete repo pool")
+		return
+	}
+
 	log.Println(">>> Delete repo pool")
 	err := deleteRepoPool(cli, authToken, repoID, repoPoolID)
 	handleError(err)
@@ -790,13 +902,31 @@ func UpdateOrgPool() {
 }
 
 func DisableOrgPool() {
+	if orgID == "" {
+		log.Println(">>> No org ID provided, skipping disable org pool")
+		return
+	}
+	if orgPoolID == "" {
+		log.Println(">>> No org pool ID provided, skipping disable org pool")
+		return
+	}
+
 	enabled := false
 	_, err := updateOrgPool(cli, authToken, orgID, orgPoolID, params.UpdatePoolParams{Enabled: &enabled})
 	handleError(err)
 	log.Printf("org pool %s disabled", orgPoolID)
 }
 
 func WaitOrgPoolNoInstances() {
+	if orgID == "" {
+		log.Println(">>> No org ID provided, skipping wait for org pool no instances")
+		return
+	}
+	if orgPoolID == "" {
+		log.Println(">>> No org pool ID provided, skipping wait for org pool no instances")
+		return
+	}
+
 	for {
 		log.Println(">>> Wait until org pool has no instances")
 		pool, err := getOrgPool(cli, authToken, orgID, orgPoolID)
@@ -835,7 +965,7 @@ func WaitOrgInstance(timeout time.Duration) {
 	handleError(err)
 	printResponse(instanceDetails)
 
-	log.Fatalf("Failed to wait for org instance to be ready")
+	panic("Failed to wait for org instance to be ready")
 }
 
 func ListOrgInstances() {
@@ -846,13 +976,27 @@ func ListOrgInstances() {
 }
 
 func DeleteOrg() {
+	if orgID == "" {
+		log.Println(">>> No org ID provided, skipping delete org")
+		return
+	}
+
 	log.Println(">>> Delete org")
 	err := deleteOrg(cli, authToken, orgID)
 	handleError(err)
 	log.Printf("org %s deleted", orgID)
 }
 
 func DeleteOrgPool() {
+	if orgID == "" {
+		log.Println(">>> No org ID provided, skipping delete org pool")
+		return
+	}
+	if orgPoolID == "" {
+		log.Println(">>> No org pool ID provided, skipping delete org pool")
+		return
+	}
+
 	log.Println(">>> Delete org pool")
 	err := deleteOrgPool(cli, authToken, orgID, orgPoolID)
 	handleError(err)
@@ -877,6 +1021,11 @@ func GetInstance() {
 }
 
 func DeleteInstance(name string) {
+	if name == "" {
+		log.Println(">>> No instance name provided, skipping delete instance")
+		return
+	}
+
 	err := deleteInstance(cli, authToken, name)
 	for {
 		log.Printf(">>> Wait until instance %s is deleted", name)
@@ -954,6 +1103,11 @@ func GetPool() {
 }
 
 func DeletePool() {
+	if poolID == "" {
+		log.Println(">>> No pool ID provided, skipping delete pool")
+		return
+	}
+
 	log.Println(">>> Delete pool")
 	err := deletePool(cli, authToken, poolID)
 	handleError(err)
@@ -968,6 +1122,13 @@ func ListPoolInstances() {
 }
 
 func main() {
+	/////////////
+	// Cleanup //
+	/////////////
+	defer GhOrgRunnersCleanup()
+	defer GhRepoRunnersCleanup()
+	defer GracefulCleanup()
+
 	//////////////////
 	// initialize cli /
 	//////////////////
@@ -1049,23 +1210,4 @@ func main() {
 	UpdatePool()
 	GetPool()
 	ListPoolInstances()
-
-	/////////////
-	// Cleanup //
-	/////////////
-	DisableRepoPool()
-	DisableOrgPool()
-
-	DeleteInstance(repoInstanceName)
-	DeleteInstance(orgInstanceName)
-
-	WaitRepoPoolNoInstances()
-	WaitOrgPoolNoInstances()
-
-	DeleteRepoPool()
-	DeleteOrgPool()
-	DeletePool()
-
-	DeleteRepo()
-	DeleteOrg()
 }