Skip to content

Commit

Permalink
agent update build steps
Browse files Browse the repository at this point in the history
  • Loading branch information
@bradrydzewski
bradrydzewski committed Apr 1, 2017
1 parent aa3fc51 commit 6c11444
Show file tree
Hide file tree
Showing 18 changed files with 508 additions and 170 deletions.
94 changes: 82 additions & 12 deletions drone/agent/agent.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,13 @@ package agent

import (
"context"
"encoding/json"
"io"
"io/ioutil"
"log"
"math"
"net/url"
"strconv"
"sync"
"time"

Expand Down Expand Up @@ -189,9 +192,9 @@ func run(ctx context.Context, client rpc.Peer, filter rpc.Filter) error {

state := rpc.State{}
state.Started = time.Now().Unix()
err = client.Update(context.Background(), work.ID, state)
err = client.Init(context.Background(), work.ID, state)
if err != nil {
log.Printf("pipeline: error updating pipeline status: %s: %s", work.ID, err)
log.Printf("pipeline: error signaling pipeline init: %s: %s", work.ID, err)
}

var uploads sync.WaitGroup
Expand All @@ -201,9 +204,31 @@ func run(ctx context.Context, client rpc.Peer, filter rpc.Filter) error {
return rerr
}
uploads.Add(1)
writer := rpc.NewLineWriter(client, work.ID, proc.Alias)
rlimit := io.LimitReader(part, maxLogsUpload)
io.Copy(writer, rlimit)

var secrets []string
for _, secret := range work.Config.Secrets {
if secret.Mask {
secrets = append(secrets, secret.Value)
}
}

limitedPart := io.LimitReader(part, maxLogsUpload)
logstream := rpc.NewLineWriter(client, work.ID, proc.Alias, secrets...)
io.Copy(logstream, limitedPart)

file := &rpc.File{}
file.Mime = "application/json+logs"
file.Proc = proc.Alias
file.Name = "logs.json"
file.Data, _ = json.Marshal(logstream.Lines())
file.Size = len(file.Data)
file.Time = time.Now().Unix()

if serr := client.Upload(context.Background(), work.ID, file); serr != nil {
log.Printf("pipeline: cannot upload logs: %s: %s: %s", work.ID, file.Mime, serr)
} else {
log.Printf("pipeline: finish uploading logs: %s: step %s: %s", file.Mime, work.ID, proc.Alias)
}

defer func() {
log.Printf("pipeline: finish uploading logs: %s: step %s", work.ID, proc.Alias)
Expand All @@ -214,18 +239,62 @@ func run(ctx context.Context, client rpc.Peer, filter rpc.Filter) error {
if rerr != nil {
return nil
}
rlimit = io.LimitReader(part, maxFileUpload)
mime := part.Header().Get("Content-Type")
if serr := client.Upload(context.Background(), work.ID, mime, rlimit); serr != nil {
log.Printf("pipeline: cannot upload artifact: %s: %s: %s", work.ID, mime, serr)
// TODO should be configurable
limitedPart = io.LimitReader(part, maxFileUpload)
file = &rpc.File{}
file.Mime = part.Header().Get("Content-Type")
file.Proc = proc.Alias
file.Name = part.FileName()
file.Data, _ = ioutil.ReadAll(limitedPart)
file.Size = len(file.Data)
file.Time = time.Now().Unix()

if serr := client.Upload(context.Background(), work.ID, file); serr != nil {
log.Printf("pipeline: cannot upload artifact: %s: %s: %s", work.ID, file.Mime, serr)
} else {
log.Printf("pipeline: finish uploading artifact: %s: step %s: %s", file.Mime, work.ID, proc.Alias)
}
return nil
})

defaultTracer := pipeline.TraceFunc(func(state *pipeline.State) error {
procState := rpc.State{
Proc: state.Pipeline.Step.Alias,
Exited: state.Process.Exited,
ExitCode: state.Process.ExitCode,
Started: time.Now().Unix(), // TODO do not do this
Finished: time.Now().Unix(),
}
defer func() {
if uerr := client.Update(context.Background(), work.ID, procState); uerr != nil {
log.Printf("Pipeine: error updating pipeline step status: %s: %s: %s", work.ID, procState.Proc, uerr)
}
}()
if state.Process.Exited {
return nil
}
if state.Pipeline.Step.Environment == nil {
state.Pipeline.Step.Environment = map[string]string{}
}
state.Pipeline.Step.Environment["CI_BUILD_STATUS"] = "success"
state.Pipeline.Step.Environment["CI_BUILD_STARTED"] = strconv.FormatInt(state.Pipeline.Time, 10)
state.Pipeline.Step.Environment["CI_BUILD_FINISHED"] = strconv.FormatInt(time.Now().Unix(), 10)

state.Pipeline.Step.Environment["CI_JOB_STATUS"] = "success"
state.Pipeline.Step.Environment["CI_JOB_STARTED"] = strconv.FormatInt(state.Pipeline.Time, 10)
state.Pipeline.Step.Environment["CI_JOB_FINISHED"] = strconv.FormatInt(time.Now().Unix(), 10)

if state.Pipeline.Error != nil {
state.Pipeline.Step.Environment["CI_BUILD_STATUS"] = "failure"
state.Pipeline.Step.Environment["CI_JOB_STATUS"] = "failure"
}
return nil
})

err = pipeline.New(work.Config,
pipeline.WithContext(ctx),
pipeline.WithLogger(defaultLogger),
pipeline.WithTracer(pipeline.DefaultTracer),
pipeline.WithTracer(defaultTracer),
pipeline.WithEngine(engine),
).Run()

Expand All @@ -247,9 +316,10 @@ func run(ctx context.Context, client rpc.Peer, filter rpc.Filter) error {
log.Printf("pipeline: execution complete: %s", work.ID)

uploads.Wait()
err = client.Update(context.Background(), work.ID, state)

err = client.Done(context.Background(), work.ID, state)
if err != nil {
log.Printf("Pipeine: error updating pipeline status: %s: %s", work.ID, err)
log.Printf("Pipeine: error signaling pipeline done: %s: %s", work.ID, err)
}

return nil
Expand Down
61 changes: 31 additions & 30 deletions model/build.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,36 +2,37 @@ package model

// swagger:model build
type Build struct {
ID int64 `json:"id" meddler:"build_id,pk"`
RepoID int64 `json:"-" meddler:"build_repo_id"`
Number int `json:"number" meddler:"build_number"`
Parent int `json:"parent" meddler:"build_parent"`
Event string `json:"event" meddler:"build_event"`
Status string `json:"status" meddler:"build_status"`
Error string `json:"error" meddler:"build_error"`
Enqueued int64 `json:"enqueued_at" meddler:"build_enqueued"`
Created int64 `json:"created_at" meddler:"build_created"`
Started int64 `json:"started_at" meddler:"build_started"`
Finished int64 `json:"finished_at" meddler:"build_finished"`
Deploy string `json:"deploy_to" meddler:"build_deploy"`
Commit string `json:"commit" meddler:"build_commit"`
Branch string `json:"branch" meddler:"build_branch"`
Ref string `json:"ref" meddler:"build_ref"`
Refspec string `json:"refspec" meddler:"build_refspec"`
Remote string `json:"remote" meddler:"build_remote"`
Title string `json:"title" meddler:"build_title"`
Message string `json:"message" meddler:"build_message"`
Timestamp int64 `json:"timestamp" meddler:"build_timestamp"`
Sender string `json:"sender" meddler:"build_sender"`
Author string `json:"author" meddler:"build_author"`
Avatar string `json:"author_avatar" meddler:"build_avatar"`
Email string `json:"author_email" meddler:"build_email"`
Link string `json:"link_url" meddler:"build_link"`
Signed bool `json:"signed" meddler:"build_signed"` // deprecate
Verified bool `json:"verified" meddler:"build_verified"` // deprecate
Reviewer string `json:"reviewed_by" meddler:"build_reviewer"`
Reviewed int64 `json:"reviewed_at" meddler:"build_reviewed"`
Jobs []*Job `json:"jobs,omitempty" meddler:"-"`
ID int64 `json:"id" meddler:"build_id,pk"`
RepoID int64 `json:"-" meddler:"build_repo_id"`
Number int `json:"number" meddler:"build_number"`
Parent int `json:"parent" meddler:"build_parent"`
Event string `json:"event" meddler:"build_event"`
Status string `json:"status" meddler:"build_status"`
Error string `json:"error" meddler:"build_error"`
Enqueued int64 `json:"enqueued_at" meddler:"build_enqueued"`
Created int64 `json:"created_at" meddler:"build_created"`
Started int64 `json:"started_at" meddler:"build_started"`
Finished int64 `json:"finished_at" meddler:"build_finished"`
Deploy string `json:"deploy_to" meddler:"build_deploy"`
Commit string `json:"commit" meddler:"build_commit"`
Branch string `json:"branch" meddler:"build_branch"`
Ref string `json:"ref" meddler:"build_ref"`
Refspec string `json:"refspec" meddler:"build_refspec"`
Remote string `json:"remote" meddler:"build_remote"`
Title string `json:"title" meddler:"build_title"`
Message string `json:"message" meddler:"build_message"`
Timestamp int64 `json:"timestamp" meddler:"build_timestamp"`
Sender string `json:"sender" meddler:"build_sender"`
Author string `json:"author" meddler:"build_author"`
Avatar string `json:"author_avatar" meddler:"build_avatar"`
Email string `json:"author_email" meddler:"build_email"`
Link string `json:"link_url" meddler:"build_link"`
Signed bool `json:"signed" meddler:"build_signed"` // deprecate
Verified bool `json:"verified" meddler:"build_verified"` // deprecate
Reviewer string `json:"reviewed_by" meddler:"build_reviewer"`
Reviewed int64 `json:"reviewed_at" meddler:"build_reviewed"`
Jobs []*Job `json:"jobs,omitempty" meddler:"-"`
Procs []*Proc `json:"procs,omitempty" meddler:"-"`
}

type BuildGroup struct {
Expand Down
2 changes: 2 additions & 0 deletions model/proc.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package model

// ProcStore persists process information to storage.
type ProcStore interface {
ProcLoad(int64) (*Proc, error)
ProcFind(*Build, int) (*Proc, error)
ProcChild(*Build, int, string) (*Proc, error)
ProcList(*Build) ([]*Proc, error)
Expand All @@ -10,6 +11,7 @@ type ProcStore interface {
}

// Proc represents a process in the build pipeline.
// swagger:model proc
type Proc struct {
ID int64 `json:"id" meddler:"proc_id,pk"`
BuildID int64 `json:"build_id" meddler:"proc_build_id"`
Expand Down
1 change: 1 addition & 0 deletions server/hook.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -490,6 +490,7 @@ func (b *builder) Build() ([]*buildItem, error) {
// TODO ability to set global volumes for things like certs
compiler.WithVolumes(),
compiler.WithWorkspaceFromURL("/drone", b.Curr.Link),
compiler.WithMetadata(metadata),
).Compile(parsed)

for _, sec := range b.Secs {
Expand Down
Loading

0 comments on commit 6c11444

Please sign in to comment.