Skip to content
This repository was archived by the owner on Sep 9, 2020. It is now read-only.

[WIP] write partial manifest upon init failure #1280

Closed
wants to merge 1 commit into from
Closed

[WIP] write partial manifest upon init failure #1280

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions cmd/dep/ensure.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -270,7 +270,7 @@ func (cmd *ensureCommand) runDefault(ctx *dep.Ctx, args []string, p *dep.Project
if !ctx.Verbose {
logger = log.New(ioutil.Discard, "", 0)
}
return errors.WithMessage(sw.Write(p.AbsRoot, sm, true, logger), "grouped write of manifest, lock and vendor")
return errors.WithMessage(sw.Write(p.AbsRoot, sm, true, false, logger), "grouped write of manifest, lock and vendor")
}

if cmd.noVendor && cmd.dryRun {
Expand Down Expand Up @@ -299,7 +299,7 @@ func (cmd *ensureCommand) runDefault(ctx *dep.Ctx, args []string, p *dep.Project
if !ctx.Verbose {
logger = log.New(ioutil.Discard, "", 0)
}
return errors.Wrap(sw.Write(p.AbsRoot, sm, false, logger), "grouped write of manifest, lock and vendor")
return errors.Wrap(sw.Write(p.AbsRoot, sm, false, false, logger), "grouped write of manifest, lock and vendor")
}

func (cmd *ensureCommand) runVendorOnly(ctx *dep.Ctx, args []string, p *dep.Project, sm gps.SourceManager, params gps.SolveParameters) error {
Expand All @@ -325,7 +325,7 @@ func (cmd *ensureCommand) runVendorOnly(ctx *dep.Ctx, args []string, p *dep.Proj
if !ctx.Verbose {
logger = log.New(ioutil.Discard, "", 0)
}
return errors.WithMessage(sw.Write(p.AbsRoot, sm, true, logger), "grouped write of manifest, lock and vendor")
return errors.WithMessage(sw.Write(p.AbsRoot, sm, true, false, logger), "grouped write of manifest, lock and vendor")
}

func (cmd *ensureCommand) runUpdate(ctx *dep.Ctx, args []string, p *dep.Project, sm gps.SourceManager, params gps.SolveParameters) error {
Expand Down Expand Up @@ -390,7 +390,7 @@ func (cmd *ensureCommand) runUpdate(ctx *dep.Ctx, args []string, p *dep.Project,
if !ctx.Verbose {
logger = log.New(ioutil.Discard, "", 0)
}
return errors.Wrap(sw.Write(p.AbsRoot, sm, false, logger), "grouped write of manifest, lock and vendor")
return errors.Wrap(sw.Write(p.AbsRoot, sm, false, false, logger), "grouped write of manifest, lock and vendor")
}

func (cmd *ensureCommand) runAdd(ctx *dep.Ctx, args []string, p *dep.Project, sm gps.SourceManager, params gps.SolveParameters) error {
Expand Down Expand Up @@ -691,7 +691,7 @@ func (cmd *ensureCommand) runAdd(ctx *dep.Ctx, args []string, p *dep.Project, sm
if !ctx.Verbose {
logger = log.New(ioutil.Discard, "", 0)
}
if err := errors.Wrap(sw.Write(p.AbsRoot, sm, true, logger), "grouped write of manifest, lock and vendor"); err != nil {
if err := errors.Wrap(sw.Write(p.AbsRoot, sm, true, false, logger), "grouped write of manifest, lock and vendor"); err != nil {
return err
}

Expand Down
27 changes: 17 additions & 10 deletions cmd/dep/init.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ import (

const initShortHelp = `Initialize a new project with manifest and lock files`
const initLongHelp = `
Wizard

Initialize the project at filepath root by parsing its dependencies, writing
manifest and lock files, and vendoring the dependencies. If root isn't
specified, use the current directory.
Expand Down Expand Up @@ -183,10 +185,13 @@ func (cmd *initCommand) Run(ctx *dep.Ctx, args []string) error {
return errors.Wrap(err, "prepare solver")
}

partial := true
vendorBehavior := dep.VendorAlways
soln, err := s.Solve()
if err != nil {
handleAllTheFailuresOfTheWorld(err)
return err
partial = true
vendorBehavior = dep.VendorNever
}
p.Lock = dep.LockFromSolution(soln)

Expand All @@ -201,16 +206,18 @@ func (cmd *initCommand) Run(ctx *dep.Ctx, args []string) error {

p.Lock.SolveMeta.InputsDigest = s.HashInputs()

// Pass timestamp (yyyyMMddHHmmss format) as suffix to backup name.
vendorbak, err := dep.BackupVendor(vpath, time.Now().Format("20060102150405"))
if err != nil {
return err
}
if vendorbak != "" {
ctx.Err.Printf("Old vendor backed up to %v", vendorbak)
if vendorBehavior == dep.VendorAlways {
// Pass timestamp (yyyyMMddHHmmss format) as suffix to backup name.
vendorbak, err := dep.BackupVendor(vpath, time.Now().Format("20060102150405"))
if err != nil {
return err
}
if vendorbak != "" {
ctx.Err.Printf("Old vendor backed up to %v", vendorbak)
}
}

sw, err := dep.NewSafeWriter(p.Manifest, nil, p.Lock, dep.VendorAlways)
sw, err := dep.NewSafeWriter(p.Manifest, nil, p.Lock, vendorBehavior)
if err != nil {
return err
}
Expand All @@ -219,7 +226,7 @@ func (cmd *initCommand) Run(ctx *dep.Ctx, args []string) error {
if !ctx.Verbose {
logger = log.New(ioutil.Discard, "", 0)
}
if err := sw.Write(root, sm, !cmd.noExamples, logger); err != nil {
if err := sw.Write(root, sm, !cmd.noExamples, partial, logger); err != nil {
return errors.Wrap(err, "safe write of manifest and lock")
}

Expand Down
70 changes: 37 additions & 33 deletions internal/gps/solver.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -440,22 +440,20 @@ func (s *solver) Solve() (Solution, error) {
all, err := s.solve()

s.mtr.pop()
var soln solution
if err == nil {
soln = solution{
att: s.attempts,
solv: s,
}
soln.analyzerInfo = s.rd.an.Info()
soln.hd = s.HashInputs()

// Convert ProjectAtoms into LockedProjects
soln.p = make([]LockedProject, len(all))
k := 0
for pa, pl := range all {
soln.p[k] = pa2lp(pa, pl)
k++
}

soln := solution{
att: s.attempts,
solv: s,
}
soln.analyzerInfo = s.rd.an.Info()
soln.hd = s.HashInputs()

// Convert ProjectAtoms into LockedProjects
soln.p = make([]LockedProject, len(all))
k := 0
for pa, pl := range all {
soln.p[k] = pa2lp(pa, pl)
k++
}

s.traceFinish(soln, err)
Expand All @@ -465,6 +463,26 @@ func (s *solver) Solve() (Solution, error) {
return soln, err
}

func (s *solver) combine() map[atom]map[string]struct{} {
projs := make(map[atom]map[string]struct{})

// Skip the first project. It's always the root, and that shouldn't be
// included in results.
for _, sel := range s.sel.projects[1:] {
pm, exists := projs[sel.a.a]
if !exists {
pm = make(map[string]struct{})
projs[sel.a.a] = pm
}

for _, path := range sel.a.pl {
pm[path] = struct{}{}
}
}

return projs
}

// solve is the top-level loop for the solving process.
func (s *solver) solve() (map[atom]map[string]struct{}, error) {
// Main solving loop
Expand Down Expand Up @@ -496,7 +514,7 @@ func (s *solver) solve() (map[atom]map[string]struct{}, error) {
// backtracking succeeded, move to the next unselected id
continue
}
return nil, err
return s.combine(), err
}

if queue.current() == nil {
Expand Down Expand Up @@ -544,7 +562,7 @@ func (s *solver) solve() (map[atom]map[string]struct{}, error) {
continue
}
s.mtr.pop()
return nil, err
return s.combine(), err
}
s.selectAtom(nawp, true)
// We don't add anything to the stack of version queues because the
Expand All @@ -556,22 +574,8 @@ func (s *solver) solve() (map[atom]map[string]struct{}, error) {

// Getting this far means we successfully found a solution. Combine the
// selected projects and packages.
projs := make(map[atom]map[string]struct{})

// Skip the first project. It's always the root, and that shouldn't be
// included in results.
for _, sel := range s.sel.projects[1:] {
pm, exists := projs[sel.a.a]
if !exists {
pm = make(map[string]struct{})
projs[sel.a.a] = pm
}

for _, path := range sel.a.pl {
pm[path] = struct{}{}
}
}
return projs, nil
return s.combine(), nil
}

// selectRoot is a specialized selectAtom, used solely to initially
Expand Down
12 changes: 11 additions & 1 deletion txn_writer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,13 @@ var exampleTOML = []byte(`

`)

var partialTOML = []byte(`
# The solver has failed (probably due to conflicting constraints).
# The output below is a partial TOML file which represents the closest
# solution that could be found.

`)

// String added on top of lock file
var lockFileComment = []byte(`# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.

Expand Down Expand Up @@ -261,7 +268,7 @@ func (sw SafeWriter) validate(root string, sm gps.SourceManager) error {
// operations succeeded. It also does its best to roll back if any moves fail.
// This mostly guarantees that dep cannot exit with a partial write that would
// leave an undefined state on disk.
func (sw *SafeWriter) Write(root string, sm gps.SourceManager, examples bool, logger *log.Logger) error {
func (sw *SafeWriter) Write(root string, sm gps.SourceManager, examples bool, partial bool, logger *log.Logger) error {
err := sw.validate(root, sm)
if err != nil {
return err
Expand Down Expand Up @@ -295,6 +302,9 @@ func (sw *SafeWriter) Write(root string, sm gps.SourceManager, examples bool, lo
if examples {
initOutput = exampleTOML
}
if partial {
initOutput = append(partialTOML, initOutput...)
}

if err = ioutil.WriteFile(filepath.Join(td, ManifestName), append(initOutput, tb...), 0666); err != nil {
return errors.Wrap(err, "failed to write manifest file to temp dir")
Expand Down
28 changes: 14 additions & 14 deletions txn_writer_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ func TestSafeWriter_BadInput_MissingRoot(t *testing.T) {
defer pc.Release()

sw, _ := NewSafeWriter(nil, nil, nil, VendorOnChanged)
err := sw.Write("", pc.SourceManager, true, discardLogger())
err := sw.Write("", pc.SourceManager, true, false, discardLogger())

if err == nil {
t.Fatal("should have errored without a root path, but did not")
Expand All @@ -44,7 +44,7 @@ func TestSafeWriter_BadInput_MissingSourceManager(t *testing.T) {
pc.Load()

sw, _ := NewSafeWriter(nil, nil, pc.Project.Lock, VendorAlways)
err := sw.Write(pc.Project.AbsRoot, nil, true, discardLogger())
err := sw.Write(pc.Project.AbsRoot, nil, true, false, discardLogger())

if err == nil {
t.Fatal("should have errored without a source manager when forceVendor is true, but did not")
Expand Down Expand Up @@ -92,7 +92,7 @@ func TestSafeWriter_BadInput_NonexistentRoot(t *testing.T) {
sw, _ := NewSafeWriter(nil, nil, nil, VendorOnChanged)

missingroot := filepath.Join(pc.Project.AbsRoot, "nonexistent")
err := sw.Write(missingroot, pc.SourceManager, true, discardLogger())
err := sw.Write(missingroot, pc.SourceManager, true, false, discardLogger())

if err == nil {
t.Fatal("should have errored with nonexistent dir for root path, but did not")
Expand All @@ -110,7 +110,7 @@ func TestSafeWriter_BadInput_RootIsFile(t *testing.T) {
sw, _ := NewSafeWriter(nil, nil, nil, VendorOnChanged)

fileroot := pc.CopyFile("fileroot", "txn_writer/badinput_fileroot")
err := sw.Write(fileroot, pc.SourceManager, true, discardLogger())
err := sw.Write(fileroot, pc.SourceManager, true, false, discardLogger())

if err == nil {
t.Fatal("should have errored when root path is a file, but did not")
Expand Down Expand Up @@ -145,7 +145,7 @@ func TestSafeWriter_Manifest(t *testing.T) {
}

// Write changes
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, discardLogger())
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, false, discardLogger())
h.Must(errors.Wrap(err, "SafeWriter.Write failed"))

// Verify file system changes
Expand Down Expand Up @@ -190,7 +190,7 @@ func TestSafeWriter_ManifestAndUnmodifiedLock(t *testing.T) {
}

// Write changes
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, discardLogger())
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, false, discardLogger())
h.Must(errors.Wrap(err, "SafeWriter.Write failed"))

// Verify file system changes
Expand Down Expand Up @@ -235,7 +235,7 @@ func TestSafeWriter_ManifestAndUnmodifiedLockWithForceVendor(t *testing.T) {
}

// Write changes
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, discardLogger())
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, false, discardLogger())
h.Must(errors.Wrap(err, "SafeWriter.Write failed"))

// Verify file system changes
Expand Down Expand Up @@ -285,7 +285,7 @@ func TestSafeWriter_ModifiedLock(t *testing.T) {
}

// Write changes
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, discardLogger())
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, false, discardLogger())
h.Must(errors.Wrap(err, "SafeWriter.Write failed"))

// Verify file system changes
Expand Down Expand Up @@ -335,7 +335,7 @@ func TestSafeWriter_ModifiedLockSkipVendor(t *testing.T) {
}

// Write changes
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, discardLogger())
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, false, discardLogger())
h.Must(errors.Wrap(err, "SafeWriter.Write failed"))

// Verify file system changes
Expand Down Expand Up @@ -363,7 +363,7 @@ func TestSafeWriter_ForceVendorWhenVendorAlreadyExists(t *testing.T) {
pc.Load()

sw, _ := NewSafeWriter(nil, pc.Project.Lock, pc.Project.Lock, VendorAlways)
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, discardLogger())
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, false, discardLogger())
h.Must(errors.Wrap(err, "SafeWriter.Write failed"))

// Verify prepared actions
Expand All @@ -381,7 +381,7 @@ func TestSafeWriter_ForceVendorWhenVendorAlreadyExists(t *testing.T) {
t.Fatal("Expected the payload to contain the vendor directory ")
}

err = sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, discardLogger())
err = sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, false, discardLogger())
h.Must(errors.Wrap(err, "SafeWriter.Write failed"))

// Verify file system changes
Expand Down Expand Up @@ -431,7 +431,7 @@ func TestSafeWriter_NewLock(t *testing.T) {
}

// Write changes
err = sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, discardLogger())
err = sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, false, discardLogger())
h.Must(errors.Wrap(err, "SafeWriter.Write failed"))

// Verify file system changes
Expand Down Expand Up @@ -478,7 +478,7 @@ func TestSafeWriter_NewLockSkipVendor(t *testing.T) {
}

// Write changes
err = sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, discardLogger())
err = sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, false, discardLogger())
h.Must(errors.Wrap(err, "SafeWriter.Write failed"))

// Verify file system changes
Expand Down Expand Up @@ -571,7 +571,7 @@ func TestSafeWriter_VendorDotGitPreservedWithForceVendor(t *testing.T) {
t.Fatal("Expected the payload to contain the vendor directory")
}

err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, discardLogger())
err := sw.Write(pc.Project.AbsRoot, pc.SourceManager, true, false, discardLogger())
h.Must(errors.Wrap(err, "SafeWriter.Write failed"))

// Verify file system changes
Expand Down