Skip to content

Commit 20b5f1d

Browse files
ayu-devtronayu-devtron
authored
Merge pull request #112 from devtron-labs/cli-output-propagate
fix: propagate cli output and save
2 parents c5916b4 + 5a819d6 commit 20b5f1d

10 files changed

+189
-147
lines changed

‎pkg/RepoManages.go

Lines changed: 22 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
"github.com/devtron-labs/git-sensor/internals/sql"
2626
"github.com/devtron-labs/git-sensor/internals/util"
2727
"github.com/devtron-labs/git-sensor/pkg/git"
28+
util2 "github.com/devtron-labs/git-sensor/util"
2829
_ "github.com/robfig/cron/v3"
2930
"go.uber.org/zap"
3031
"strings"
@@ -202,12 +203,16 @@ func (impl RepoManagerImpl) updatePipelineMaterialCommit(gitCtx git.GitContext,
202203

203204
fetchCount := impl.configuration.GitHistoryCount
204205
var repository *git.GitRepository
205-
commits, err := impl.repositoryManager.ChangesSinceByRepository(gitCtx, repository, pipelineMaterial.Value, "", "", fetchCount, material.CheckoutLocation, true)
206+
commits, errMsg, err := impl.repositoryManager.ChangesSinceByRepository(gitCtx, repository, pipelineMaterial.Value, "", "", fetchCount, material.CheckoutLocation, true)
206207
//commits, err := impl.FetchChanges(pipelineMaterial.Id, "", "", 0)
207208
if gitCtx.Err() != nil {
208209
impl.logger.Errorw("context error in getting commits", "err", gitCtx.Err())
209210
return gitCtx.Err()
210-
} else if err == nil {
211+
} else if err != nil {
212+
pipelineMaterial.Errored = true
213+
pipelineMaterial.ErrorMsg = util2.BuildDisplayErrorMessage(errMsg, err)
214+
pipelineMaterial.LastSeenHash = ""
215+
} else {
211216
impl.logger.Infow("commits found", "commit", commits)
212217
b, err := json.Marshal(commits)
213218
if err == nil {
@@ -226,10 +231,6 @@ func (impl RepoManagerImpl) updatePipelineMaterialCommit(gitCtx git.GitContext,
226231
pipelineMaterial.ErrorMsg = err.Error()
227232
pipelineMaterial.LastSeenHash = ""
228233
}
229-
} else {
230-
pipelineMaterial.Errored = true
231-
pipelineMaterial.ErrorMsg = err.Error()
232-
pipelineMaterial.LastSeenHash = ""
233234
}
234235
materialCommits = append(materialCommits, pipelineMaterial)
235236
}
@@ -366,17 +367,17 @@ func (impl RepoManagerImpl) checkoutMaterial(gitCtx git.GitContext, material *sq
366367

367368
checkoutLocationForFetching := impl.repositoryManager.GetCheckoutLocation(gitCtx, material, gitProvider.Url, checkoutPath)
368369

369-
err = impl.repositoryManager.Add(gitCtx, material.GitProviderId, checkoutPath, material.Url, gitProvider.AuthMode, gitProvider.SshPrivateKey)
370+
errMsg, err := impl.repositoryManager.Add(gitCtx, material.GitProviderId, checkoutPath, material.Url, gitProvider.AuthMode, gitProvider.SshPrivateKey)
370371
if gitCtx.Err() != nil {
371372
impl.logger.Errorw("context error in git checkout", "err", gitCtx.Err())
372373
return material, gitCtx.Err()
373-
} else if err == nil {
374-
material.CheckoutLocation = checkoutLocationForFetching
375-
material.CheckoutStatus = true
376-
} else {
374+
} else if err != nil {
377375
material.CheckoutStatus = false
378376
material.CheckoutMsgAny = err.Error()
379-
material.FetchErrorMessage = err.Error()
377+
material.FetchErrorMessage = util2.BuildDisplayErrorMessage(errMsg, err)
378+
} else {
379+
material.CheckoutLocation = checkoutLocationForFetching
380+
material.CheckoutStatus = true
380381
}
381382
err = impl.materialRepository.Update(material)
382383
if err != nil {
@@ -668,19 +669,20 @@ func (impl RepoManagerImpl) GetLatestCommitForBranch(gitCtx git.GitContext, pipe
668669

669670
gitCtx = gitCtx.WithCredentials(userName, password).
670671
WithTLSData(gitMaterial.GitProvider.CaCert, gitMaterial.GitProvider.TlsKey, gitMaterial.GitProvider.TlsCert, gitMaterial.GitProvider.EnableTLSVerification)
671-
updated, repo, err := impl.repositoryManager.Fetch(gitCtx, gitMaterial.Url, gitMaterial.CheckoutLocation)
672+
updated, repo, errMsg, err := impl.repositoryManager.Fetch(gitCtx, gitMaterial.Url, gitMaterial.CheckoutLocation)
672673
if !updated {
673674
impl.logger.Warn("repository is up to date")
674675
}
675-
if err == nil {
676-
gitMaterial.CheckoutStatus = true
677-
} else {
676+
if err != nil {
678677
gitMaterial.CheckoutStatus = false
679678
gitMaterial.CheckoutMsgAny = err.Error()
680-
gitMaterial.FetchErrorMessage = err.Error()
679+
gitMaterial.FetchErrorMessage = util2.BuildDisplayErrorMessage(errMsg, err)
681680

682681
impl.logger.Errorw("error in fetching the repository ", "gitMaterial", gitMaterial, "err", err)
683682
return nil, err
683+
} else {
684+
gitMaterial.CheckoutStatus = true
685+
684686
}
685687

686688
err = impl.materialRepository.Update(gitMaterial)
@@ -699,7 +701,7 @@ func (impl RepoManagerImpl) GetLatestCommitForBranch(gitCtx git.GitContext, pipe
699701
return nil, err
700702
}
701703

702-
commits, err := impl.repositoryManager.ChangesSinceByRepository(gitCtx, repo, branchName, "", "", 1, gitMaterial.CheckoutLocation, false)
704+
commits, _, err := impl.repositoryManager.ChangesSinceByRepository(gitCtx, repo, branchName, "", "", 1, gitMaterial.CheckoutLocation, false)
703705

704706
if commits == nil {
705707
return nil, err
@@ -749,9 +751,9 @@ func (impl RepoManagerImpl) GetCommitMetadataForPipelineMaterial(gitCtx git.GitC
749751
impl.locker.ReturnLocker(gitMaterial.Id)
750752
}()
751753
var repository *git.GitRepository
752-
commits, err := impl.repositoryManager.ChangesSinceByRepository(gitCtx, repository, branchName, "", gitHash, 1, gitMaterial.CheckoutLocation, true)
754+
commits, errMsg, err := impl.repositoryManager.ChangesSinceByRepository(gitCtx, repository, branchName, "", gitHash, 1, gitMaterial.CheckoutLocation, true)
753755
if err != nil {
754-
if strings.Contains(err.Error(), git.NO_COMMIT_CUSTOM_ERROR_MESSAGE) {
756+
if strings.Contains(errMsg, git.NO_COMMIT_CUSTOM_ERROR_MESSAGE) {
755757
impl.logger.Warnw("No commit found for given hash", "hash", gitHash, "branchName", branchName)
756758
return nil, nil
757759
}

‎pkg/git/GitBaseManager.go

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -37,15 +37,15 @@ type GitManager interface {
3737
// GetCommitStats retrieves the stats for the given commit vs its parent
3838
GetCommitStats(gitCtx GitContext, commit GitCommit, checkoutPath string) (FileStats, error)
3939
// GetCommitIterator returns an iterator for the provided git repo and iterator request describing the commits to fetch
40-
GetCommitIterator(gitCtx GitContext, repository *GitRepository, iteratorRequest IteratorRequest) (CommitIterator, error)
40+
GetCommitIterator(gitCtx GitContext, repository *GitRepository, iteratorRequest IteratorRequest) (commitIterator CommitIterator, cliOutput string, errMsg string, err error)
4141
// GetCommitForHash retrieves the commit reference for given tag
4242
GetCommitForHash(gitCtx GitContext, checkoutPath, commitHash string) (GitCommit, error)
4343
// GetCommitsForTag retrieves the commit reference for given tag
4444
GetCommitsForTag(gitCtx GitContext, checkoutPath, tag string) (GitCommit, error)
4545
// OpenRepoPlain opens a new git repo at the given path
4646
OpenRepoPlain(checkoutPath string) (*GitRepository, error)
4747
// Init initializes a git repo
48-
Init(gitCtx GitContext, rootDir string, remoteUrl string, isBare bool) error
48+
Init(gitCtx GitContext, rootDir string, remoteUrl string, isBare bool) (string, error)
4949
}
5050

5151
// GitManagerBase Base methods which will be available to all implementation of the parent interface
@@ -120,7 +120,7 @@ func (impl *GitManagerBaseImpl) Fetch(gitCtx GitContext, rootDir string) (respon
120120
}
121121
defer commonLibGitManager.DeleteTlsFiles(tlsPathInfo)
122122
output, errMsg, err := impl.runCommandWithCred(cmd, gitCtx.Username, gitCtx.Password, tlsPathInfo)
123-
if strings.Contains(output, LOCK_REF_MESSAGE) {
123+
if strings.Contains(output, util.LOCK_REF_MESSAGE) {
124124
impl.logger.Info("error in fetch, pruning local refs and retrying", "rootDir", rootDir)
125125
// running git remote prune origin and retrying fetch. gitHub issue - https://github.com/devtron-labs/devtron/issues/4605
126126
pruneCmd, pruneCmdCancel := impl.createCmdWithContext(gitCtx, "git", "-C", rootDir, "remote", "prune", "origin")
@@ -176,7 +176,7 @@ func (impl *GitManagerBaseImpl) LogMergeBase(gitCtx GitContext, rootDir, from st
176176

177177
func (impl *GitManagerBaseImpl) runCommandWithCred(cmd *exec.Cmd, userName, password string, tlsPathInfo *commonLibGitManager.TlsPathInfo) (response, errMsg string, err error) {
178178
cmd.Env = append(os.Environ(),
179-
fmt.Sprintf("GIT_ASKPASS=%s", GIT_ASK_PASS),
179+
fmt.Sprintf("GIT_ASKPASS=%s", util.GIT_ASK_PASS),
180180
fmt.Sprintf("GIT_USERNAME=%s", userName),
181181
fmt.Sprintf("GIT_PASSWORD=%s", password),
182182
)
@@ -200,13 +200,18 @@ func (impl *GitManagerBaseImpl) runCommand(cmd *exec.Cmd) (response, errMsg stri
200200
output = strings.TrimSpace(output)
201201
if err != nil {
202202
impl.logger.Errorw("error in git cli operation", "msg", string(outBytes), "err", err)
203+
errMsg = output
203204
exErr, ok := err.(*exec.ExitError)
204205
if !ok {
205-
return output, string(outBytes), err
206+
return output, errMsg, err
206207
}
207-
if strings.Contains(output, AUTHENTICATION_FAILED_ERROR) {
208-
impl.logger.Errorw("authentication failed", "msg", string(outBytes), "err", err.Error())
209-
return output, "authentication failed", errors.New("authentication failed")
208+
customErrMsg := util.GetErrMsgFromCliMessage(output, err)
209+
if customErrMsg != "" {
210+
impl.logger.Errorw(customErrMsg, "msg", string(outBytes), "err", err.Error())
211+
return output, customErrMsg, errors.New(customErrMsg)
212+
}
213+
if exErr.Stderr == nil {
214+
return output, errMsg, err
210215
}
211216
errOutput := string(exErr.Stderr)
212217
return output, errOutput, err

‎pkg/git/GitCliManager.go

Lines changed: 22 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
package git
1818

1919
import (
20-
"errors"
2120
"go.uber.org/zap"
2221
"gopkg.in/src-d/go-billy.v4/osfs"
2322
"os"
@@ -39,23 +38,19 @@ func NewGitCliManagerImpl(baseManager GitManagerBase, logger *zap.SugaredLogger)
3938
}
4039
}
4140

42-
const (
43-
GIT_ASK_PASS = "/git-ask-pass.sh"
44-
AUTHENTICATION_FAILED_ERROR = "Authentication failed"
45-
LOCK_REF_MESSAGE = "cannot lock ref"
46-
)
47-
48-
func (impl *GitCliManagerImpl) Init(gitCtx GitContext, rootDir string, remoteUrl string, isBare bool) error {
41+
func (impl *GitCliManagerImpl) Init(gitCtx GitContext, rootDir string, remoteUrl string, isBare bool) (string, error) {
4942
//-----------------
5043

51-
err := os.MkdirAll(rootDir, 0755)
44+
var err error
45+
var errMsg string
46+
err = os.MkdirAll(rootDir, 0755)
5247
if err != nil {
53-
return err
48+
return "", err
5449
}
5550

56-
err = impl.GitInit(gitCtx, rootDir)
51+
errMsg, err = impl.GitInit(gitCtx, rootDir)
5752
if err != nil {
58-
return err
53+
return errMsg, err
5954
}
6055
return impl.GitCreateRemote(gitCtx, rootDir, remoteUrl)
6156

@@ -80,16 +75,16 @@ func (impl *GitCliManagerImpl) GetCommitForHash(gitCtx GitContext, checkoutPath,
8075

8176
return impl.GitShow(gitCtx, checkoutPath, commitHash)
8277
}
83-
func (impl *GitCliManagerImpl) GetCommitIterator(gitCtx GitContext, repository *GitRepository, iteratorRequest IteratorRequest) (CommitIterator, error) {
78+
func (impl *GitCliManagerImpl) GetCommitIterator(gitCtx GitContext, repository *GitRepository, iteratorRequest IteratorRequest) (commitIterator CommitIterator, cliOutput string, errMsg string, err error) {
8479

85-
commits, err := impl.GetCommits(gitCtx, iteratorRequest.BranchRef, iteratorRequest.Branch, repository.rootDir, iteratorRequest.CommitCount, iteratorRequest.FromCommitHash, iteratorRequest.ToCommitHash)
80+
commits, cliOutput, errMsg, err := impl.GetCommits(gitCtx, iteratorRequest.BranchRef, iteratorRequest.Branch, repository.rootDir, iteratorRequest.CommitCount, iteratorRequest.FromCommitHash, iteratorRequest.ToCommitHash)
8681
if err != nil {
8782
impl.logger.Errorw("error in fetching commits for", "err", err, "path", repository.rootDir)
88-
return nil, err
83+
return nil, cliOutput, errMsg, err
8984
}
9085
return &CommitCliIterator{
9186
commits: commits,
92-
}, nil
87+
}, "", "", nil
9388
}
9489

9590
func openGitRepo(path string) error {
@@ -103,39 +98,36 @@ func openGitRepo(path string) error {
10398
}
10499
return nil
105100
}
106-
func (impl *GitCliManagerImpl) GitInit(gitCtx GitContext, rootDir string) error {
101+
func (impl *GitCliManagerImpl) GitInit(gitCtx GitContext, rootDir string) (string, error) {
107102
impl.logger.Debugw("git", "-C", rootDir, "init")
108103
output, errMsg, err := impl.GitManagerBase.ExecuteCustomCommand(gitCtx, "git", "-C", rootDir, "init")
109104
impl.logger.Debugw("root", rootDir, "opt", output, "errMsg", errMsg, "error", err)
110-
return err
105+
return errMsg, err
111106
}
112107

113-
func (impl *GitCliManagerImpl) GitCreateRemote(gitCtx GitContext, rootDir string, url string) error {
108+
func (impl *GitCliManagerImpl) GitCreateRemote(gitCtx GitContext, rootDir string, url string) (string, error) {
114109
impl.logger.Debugw("git", "-C", rootDir, "remote", "add", "origin", url)
115110
output, errMsg, err := impl.GitManagerBase.ExecuteCustomCommand(gitCtx, "git", "-C", rootDir, "remote", "add", "origin", url)
116111
impl.logger.Debugw("url", url, "opt", output, "errMsg", errMsg, "error", err)
117-
return err
112+
return errMsg, err
118113
}
119114

120-
func (impl *GitCliManagerImpl) GetCommits(gitCtx GitContext, branchRef string, branch string, rootDir string, numCommits int, from string, to string) ([]GitCommit, error) {
115+
func (impl *GitCliManagerImpl) GetCommits(gitCtx GitContext, branchRef string, branch string, rootDir string, numCommits int, from string, to string) (commits []GitCommit, cliOutput string, errMsg string, err error) {
121116
baseCmdArgs := []string{"-C", rootDir, "log"}
122117
rangeCmdArgs := []string{branchRef}
123118
extraCmdArgs := []string{"-n", strconv.Itoa(numCommits), "--date=iso-strict", GITFORMAT}
124119
cmdArgs := impl.getCommandForLogRange(branchRef, from, to, rangeCmdArgs, baseCmdArgs, extraCmdArgs)
125120
impl.logger.Debugw("git", cmdArgs)
126-
output, errMsg, err := impl.GitManagerBase.ExecuteCustomCommand(gitCtx, "git", cmdArgs...)
127-
impl.logger.Debugw("root", rootDir, "opt", output, "errMsg", errMsg, "error", err)
121+
cliOutput, errMsg, err = impl.GitManagerBase.ExecuteCustomCommand(gitCtx, "git", cmdArgs...)
122+
impl.logger.Debugw("root", rootDir, "cliOutput", cliOutput, "errMsg", errMsg, "error", err)
128123
if err != nil {
129-
if strings.Contains(output, NO_COMMIT_GIT_ERROR_MESSAGE) {
130-
return nil, errors.New(NO_COMMIT_CUSTOM_ERROR_MESSAGE)
131-
}
132-
return nil, err
124+
return nil, cliOutput, errMsg, err
133125
}
134-
commits, err := impl.processGitLogOutput(output)
126+
commits, err = impl.processGitLogOutput(cliOutput)
135127
if err != nil {
136-
return nil, err
128+
return nil, "", "", err
137129
}
138-
return commits, nil
130+
return commits, "", "", nil
139131
}
140132

141133
func (impl *GitCliManagerImpl) getCommandForLogRange(branchRef string, from string, to string, rangeCmdArgs []string, baseCmdArgs []string, extraCmdArgs []string) []string {

‎pkg/git/GoGitSDKManager.go

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -94,19 +94,19 @@ func (impl *GoGitSDKManagerImpl) GetCommitForHash(gitCtx GitContext, checkoutPat
9494
return gitCommit, nil
9595
}
9696

97-
func (impl *GoGitSDKManagerImpl) GetCommitIterator(gitCtx GitContext, repository *GitRepository, iteratorRequest IteratorRequest) (CommitIterator, error) {
97+
func (impl *GoGitSDKManagerImpl) GetCommitIterator(gitCtx GitContext, repository *GitRepository, iteratorRequest IteratorRequest) (commitIterator CommitIterator, cliOutput string, errMsg string, err error) {
9898

9999
ref, err := repository.Reference(plumbing.ReferenceName(iteratorRequest.BranchRef), true)
100100
if err != nil && err == plumbing.ErrReferenceNotFound {
101-
return nil, fmt.Errorf("ref not found %s branch %s", err, iteratorRequest.Branch)
101+
return nil, "", "", fmt.Errorf("ref not found %s branch %s", err, iteratorRequest.Branch)
102102
} else if err != nil {
103-
return nil, fmt.Errorf("error in getting reference %s branch %s", err, iteratorRequest.Branch)
103+
return nil, "", "", fmt.Errorf("error in getting reference %s branch %s", err, iteratorRequest.Branch)
104104
}
105105
itr, err := repository.Log(&git.LogOptions{From: ref.Hash()})
106106
if err != nil {
107-
return nil, fmt.Errorf("error in getting iterator %s branch %s", err, iteratorRequest.Branch)
107+
return nil, "", "", fmt.Errorf("error in getting iterator %s branch %s", err, iteratorRequest.Branch)
108108
}
109-
return &CommitGoGitIterator{itr}, nil
109+
return &CommitGoGitIterator{itr}, "", "", nil
110110
}
111111

112112
func (impl *GoGitSDKManagerImpl) OpenRepoPlain(checkoutPath string) (*GitRepository, error) {
@@ -119,23 +119,26 @@ func (impl *GoGitSDKManagerImpl) OpenRepoPlain(checkoutPath string) (*GitReposit
119119
return &GitRepository{Repository: r}, err
120120
}
121121

122-
func (impl *GoGitSDKManagerImpl) Init(gitCtx GitContext, rootDir string, remoteUrl string, isBare bool) error {
122+
func (impl *GoGitSDKManagerImpl) Init(gitCtx GitContext, rootDir string, remoteUrl string, isBare bool) (string, error) {
123123
//-----------------
124124

125125
err := os.MkdirAll(rootDir, 0755)
126126
if err != nil {
127-
return err
127+
return "", err
128128
}
129129

130130
repo, err := git.PlainInit(rootDir, isBare)
131131
if err != nil {
132-
return err
132+
return "", err
133133
}
134134
_, err = repo.CreateRemote(&config.RemoteConfig{
135135
Name: git.DefaultRemoteName,
136136
URLs: []string{remoteUrl},
137137
})
138-
return err
138+
if err != nil {
139+
return "", err
140+
}
141+
return "", nil
139142
}
140143

141144
func (impl *GoGitSDKManagerImpl) GetCommitStats(gitCtx GitContext, commit GitCommit, checkoutPath string) (FileStats, error) {

0 commit comments

Comments
 (0)