add ability to run github-experimental against private repos (#4508)

The existing implementation of github-experimental requires a github token for the --object-discovery subcommand (that's the only subcommand atm); however, it didn't properly use that token to clone private repos. I used the existing logic in the github/connector.go file to add enable cloning of private repositories.

Author: joeleonjr

pkg/sources/github_experimental/github_experimental.go

@@ -2,26 +2,26 @@ package github_experimental
 
 import (
 	"fmt"
-	"net/http"
 	"strings"
 
 	"github.com/go-logr/logr"
-	"github.com/google/go-github/v67/github"
 	"google.golang.org/protobuf/proto"
 	"google.golang.org/protobuf/types/known/anypb"
 
-	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/giturl"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/log"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
+	githubsource "github.com/trufflesecurity/trufflehog/v3/pkg/sources/github"
 )
 
 const (
-	SourceType = sourcespb.SourceType_SOURCE_TYPE_GITHUB_EXPERIMENTAL
+	SourceType      = sourcespb.SourceType_SOURCE_TYPE_GITHUB_EXPERIMENTAL
+	cloudV3Endpoint = "https://api.github.com"
 )
 
 type Source struct {
@@ -33,10 +33,9 @@ type Source struct {
 	useCustomContentWriter bool
 	git                    *git.Git
 	scanOptions            *git.ScanOptions
-	httpClient             *http.Client
 	log                    logr.Logger
 	conn                   *sourcespb.GitHubExperimental
-	apiClient              *github.Client
+	connector              githubsource.Connector
 
 	sources.Progress
 	sources.CommonSourceUnitUnmarshaller
@@ -81,9 +80,6 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so
 	s.jobID = jobID
 	s.verify = verify
 
-	s.httpClient = common.RetryableHTTPClientTimeout(60)
-	s.apiClient = github.NewClient(s.httpClient)
-
 	var conn sourcespb.GitHubExperimental
 	err = anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{})
 	if err != nil {
@@ -95,6 +91,32 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so
 		return fmt.Errorf("error normalizing repo: %w", err)
 	}
 
+	// Get the token from the connection
+	token := s.conn.GetToken()
+	if token == "" {
+		return fmt.Errorf("token is required for GitHub Experimental source")
+	}
+
+	// Redact token from logs for security
+	log.RedactGlobally(token)
+
+	// Create authenticated connector using the TokenConnector pattern
+	connector, err := githubsource.NewTokenConnector(
+		aCtx,
+		cloudV3Endpoint, // API endpoint
+		token,           // GitHub token
+		"",              // clonePath (empty for default)
+		true,            // authInUrl
+		func(ctx context.Context, err error) bool {
+			// Simple rate limit handler - can be enhanced later
+			return false
+		},
+	)
+	if err != nil {
+		return fmt.Errorf("could not create GitHub connector: %w", err)
+	}
+	s.connector = connector
+
 	s.repoInfoCache = newRepoInfoCache()
 
 	cfg := &git.Config{

pkg/sources/github_experimental/object_discovery.go

@@ -14,11 +14,9 @@ import (
 	"github.com/google/go-github/v67/github"
 	"github.com/k0kubun/go-ansi"
 	"github.com/schollz/progressbar/v3"
-	"golang.org/x/oauth2"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
-	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
 )
 
 // Assumption: sleeping for 60 seconds is enough to reset the secondary rate limit
@@ -100,18 +98,7 @@ func (b *backoff) getValue() int {
 	return int(b.value)
 }
 
-// Github token
-var ghToken = ""
-
-func getForksCount(owner, repoName string) (int, error) {
-	ctx := context.Background()
-	ts := oauth2.StaticTokenSource(
-		&oauth2.Token{AccessToken: ghToken},
-	)
-	tc := oauth2.NewClient(ctx, ts)
-
-	client := github.NewClient(tc)
-
+func getForksCount(ctx context.Context, client *github.Client, owner, repoName string) (int, error) {
 	repo, _, err := client.Repositories.Get(ctx, owner, repoName)
 	if err != nil {
 		return 0, err
@@ -120,22 +107,6 @@ func getForksCount(owner, repoName string) (int, error) {
 	return repo.GetForksCount(), nil
 }
 
-func getGitHubUser() (string, error) {
-	ctx := context.Background()
-	ts := oauth2.StaticTokenSource(
-		&oauth2.Token{AccessToken: ghToken},
-	)
-	tc := oauth2.NewClient(ctx, ts)
-
-	client := github.NewClient(tc)
-
-	ghUser, _, err := client.Users.Get(ctx, "")
-	if err != nil {
-		return "", err
-	}
-	return ghUser.GetLogin(), nil
-}
-
 // runGitCommand runs a git command
 func runGitCommand(args []string) ([]byte, error) {
 	cmd := exec.Command("git", args...)
@@ -322,7 +293,7 @@ func removeBySHA(existingCommits, newCommits []string, charLen int) []string {
 	return filteredCommits
 }
 
-func processCommits(ctx context.Context, needsProcessing []string, owner, repo, path string) {
+func processCommits(ctx context.Context, apiClient *github.Client, needsProcessing []string, owner, repo, path string) {
 	repoCtx := context.WithValue(ctx, "repo", repo)
 
 	startingSize := float64(len(needsProcessing))
@@ -343,10 +314,12 @@ func processCommits(ctx context.Context, needsProcessing []string, owner, repo,
 		chunk := needsProcessing[:chunkSize]
 		needsProcessing = needsProcessing[chunkSize:]
 
-		commitData, err := checkHashes(owner, repo, chunk)
+		commitData, err := checkHashes(repoCtx, apiClient, owner, repo, chunk)
 		if err != nil {
 			repoCtx.Logger().V(2).Info("Temporary error occurred in guessing commits", "error", err)
-			needsProcessing = append(needsProcessing, chunk...)
+			// Prepend the failed chunk to the FRONT of the queue for immediate retry
+			// This ensures we retry the same hashes instead of moving to the next batch
+			needsProcessing = append(chunk, needsProcessing...)
 			queryChunkSize.errorOccurred()
 			if strings.Contains(err.Error(), "You have exceeded a secondary rate limit") {
 				repoCtx.Logger().V(2).Info("Reached secondary GitHub Rate Limit. Sleeping for 60 seconds.")
@@ -391,7 +364,7 @@ type responseData struct {
 	Message string `json:"message"`
 }
 
-func checkHashes(owner, repo string, hashes []string) (map[string][]string, error) {
+func checkHashes(ctx context.Context, client *github.Client, owner, repo string, hashes []string) (map[string][]string, error) {
 	testCases := ""
 	for _, h := range hashes {
 		testCase := fmt.Sprintf(`
@@ -413,7 +386,6 @@ func checkHashes(owner, repo string, hashes []string) (map[string][]string, erro
     `, owner, repo, testCases)
 
 	headers := map[string]string{
-		"Authorization":         "Bearer " + ghToken,
 		"Content-Type":          "application/json",
 		"Github-Verified-Fetch": "true",
 		"X-Requested-With":      "XMLHttpRequest",
@@ -426,7 +398,7 @@ func checkHashes(owner, repo string, hashes []string) (map[string][]string, erro
 		return nil, fmt.Errorf("failed to marshal request body: %w", err)
 	}
 
-	req, err := http.NewRequest("POST", "https://api.github.com/graphql", bytes.NewBuffer(requestBody))
+	req, err := http.NewRequestWithContext(ctx, "POST", "https://api.github.com/graphql", bytes.NewBuffer(requestBody))
 	if err != nil {
 		return nil, fmt.Errorf("failed to create request: %w", err)
 	}
@@ -435,8 +407,9 @@ func checkHashes(owner, repo string, hashes []string) (map[string][]string, erro
 		req.Header.Set(key, value)
 	}
 
-	client := &http.Client{}
-	resp, err := client.Do(req)
+	// Use the authenticated HTTP client from the GitHub API client
+	// This client already has the Bearer token configured via OAuth2 transport
+	resp, err := client.Client().Do(req)
 	if err != nil {
 		return nil, fmt.Errorf("python request error: %w", err)
 	}
@@ -546,9 +519,6 @@ func downloadPatches(valid_cfor []string, path string) error {
 
 // scanHiddenData scans hidden data (and non-hidden data) for secrets in a GitHub repository
 func (s *Source) EnumerateAndScanAllObjects(ctx context.Context, chunksChan chan *sources.Chunk) error {
-	// assign github token to global variable
-	ghToken = s.conn.GetToken()
-
 	// set collision threshold to user input
 	collisionThreshold = float64(s.conn.CollisionThreshold)
 
@@ -564,7 +534,7 @@ func (s *Source) EnumerateAndScanAllObjects(ctx context.Context, chunksChan chan
 
 	// get repo metadata and store in cacheRepoInfo
 	repoCtx := context.WithValue(ctx, "repo", owner+"/"+repoName)
-	ghRepo, _, err := s.apiClient.Repositories.Get(repoCtx, owner, repoName)
+	ghRepo, _, err := s.connector.APIClient().Repositories.Get(repoCtx, owner, repoName)
 	if err != nil {
 		return fmt.Errorf("failed to fetch repository: %w", err)
 	}
@@ -582,20 +552,14 @@ func (s *Source) EnumerateAndScanAllObjects(ctx context.Context, chunksChan chan
 		return fmt.Errorf("failed to create .trufflehog folder in user's home directory: %w", err)
 	}
 
-	// Get GitHub User tied to token
-	ghUser, err := getGitHubUser()
-	if err != nil {
-		return fmt.Errorf("failed to get GitHub user details: %w", err)
-	}
-
 	// get the number of forks
-	forksCount, err := getForksCount(owner, repoName)
+	forksCount, err := getForksCount(repoCtx, s.connector.APIClient(), owner, repoName)
 	if err != nil {
 		return fmt.Errorf("failed to get forks count: %w", err)
 	}
 
-	// download the repo
-	path, repo, err := git.CloneRepoUsingToken(ctx, ghToken, repoURL, "", ghUser, true)
+	// download the repo using the authenticated connector
+	path, repo, err := s.connector.Clone(ctx, repoURL)
 	if err != nil {
 		return fmt.Errorf("failed to clone the repository: %w", err)
 	}
@@ -639,7 +603,7 @@ func (s *Source) EnumerateAndScanAllObjects(ctx context.Context, chunksChan chan
 	possibleCommits = removeByShortSHA(invalidCommits, possibleCommits)
 
 	// Guess all possible commit hashes
-	processCommits(ctx, possibleCommits, owner, repoName, folderPath)
+	processCommits(ctx, s.connector.APIClient(), possibleCommits, owner, repoName, folderPath)
 
 	// Read in the new commits
 	validHiddenCommits, err = readCommitsFromDisk(validHiddenCommit, folderPath)

pkg/sources/github_experimental/repo.go

@@ -2,6 +2,7 @@ package github_experimental
 
 import (
 	"fmt"
+	"regexp"
 	"strings"
 	"sync"
 
@@ -62,10 +63,15 @@ func (s *Source) cacheRepoInfo(r *github.Repository) {
 }
 
 func (s *Source) normalizeRepo(repo string) (string, error) {
-	// If there's a '/', assume it's a URL and try to normalize it.
-	if strings.ContainsRune(repo, '/') {
+	// If it's a full URL (has protocol), normalize it
+	if regexp.MustCompile(`^[a-z]+://`).MatchString(repo) {
 		return giturl.NormalizeGithubRepo(repo)
 	}
+	// If it's a repository name (contains / but not http), convert to full URL first
+	if strings.Contains(repo, "/") && !regexp.MustCompile(`^[a-z]+://`).MatchString(repo) {
+		fullURL := "https://github.com/" + repo
+		return giturl.NormalizeGithubRepo(fullURL)
+	}
 
 	return "", fmt.Errorf("no repositories found for %s", repo)
 }