diff --git a/cmd/cue/cmd/testdata/script/eval_loaderr.txtar b/cmd/cue/cmd/testdata/script/eval_loaderr.txtar index c60ad3ed233..4fad1ea4da0 100644 --- a/cmd/cue/cmd/testdata/script/eval_loaderr.txtar +++ b/cmd/cue/cmd/testdata/script/eval_loaderr.txtar @@ -1,6 +1,6 @@ -! exec cue eval non-existing . +! exec cue eval nonexisting . ! stdout . cmp stderr expect-stderr -- expect-stderr -- -cannot find package "non-existing" +cannot find package "nonexisting" diff --git a/cue/load/config.go b/cue/load/config.go index 2efd0a8d5b2..11948d04c62 100644 --- a/cue/load/config.go +++ b/cue/load/config.go @@ -27,6 +27,7 @@ import ( "cuelang.org/go/cue/errors" "cuelang.org/go/cue/token" "cuelang.org/go/internal" + "cuelang.org/go/internal/mod/modfile" ) const ( @@ -139,7 +140,7 @@ type Config struct { // if no module file was present. If non-nil, then // after calling Config.complete, modFile.Module will be // equal to Module. - modFile *modFile + modFile *modfile.File // Package defines the name of the package to be loaded. If this is not set, // the package must be uniquely defined from its context. Special values: @@ -364,6 +365,7 @@ func (c Config) complete() (cfg *Config, err error) { } else if !filepath.IsAbs(c.ModuleRoot) { c.ModuleRoot = filepath.Join(c.Dir, c.ModuleRoot) } + //c.Registry = "registry.cue.works" if c.Registry != "" { u, err := url.Parse(c.Registry) if err != nil { diff --git a/cue/load/import.go b/cue/load/import.go index b75246dcfd0..ce9fe47fe72 100644 --- a/cue/load/import.go +++ b/cue/load/import.go @@ -15,6 +15,7 @@ package load import ( + "context" "fmt" "os" pathpkg "path" @@ -22,10 +23,12 @@ import ( "sort" "strings" + "cuelang.org/go/cue/ast" "cuelang.org/go/cue/build" "cuelang.org/go/cue/errors" "cuelang.org/go/cue/token" "cuelang.org/go/internal/filetypes" + "cuelang.org/go/internal/mod/module" ) // importPkg returns details about the CUE package named by the import path, @@ -353,15 +356,25 @@ func (l *loader) absDirFromImportPath(pos token.Pos, p importPath) (absDir, name p = p[:i] default: // p[i] == '/' - name = string(p[i+1:]) + mp, _, ok := module.SplitPathVersion(string(p)) + if ok { + // import of the form: example.com/foo/bar@v1 + if i := strings.LastIndex(mp, "/"); i >= 0 { + name = mp[i+1:] + } + } else { + name = string(p[i+1:]) + } } - // TODO: fully test that name is a valid identifier. if name == "" { err = errors.Newf(pos, "empty package name in import path %q", p) } else if strings.IndexByte(name, '.') >= 0 { err = errors.Newf(pos, "cannot determine package name for %q (set explicitly with ':')", p) + } else if !ast.IsValidIdent(name) { + err = errors.Newf(pos, + "implied package identifier %q from import path %q is not valid", name, p) } // Determine the directory. @@ -381,7 +394,7 @@ func (l *loader) absDirFromImportPath(pos token.Pos, p importPath) (absDir, name absDir, err = l.externalPackageDir(p) if err != nil { // TODO why can't we use %w ? - return "", name, errors.Newf(token.NoPos, "cannot get directory for external module %q: %v", p, err) + return "", name, errors.Newf(token.NoPos, "cannot get directory for external module %q (registry %q): %v", p, l.cfg.Registry, err) } } else { absDir = filepath.Join(GenPath(l.cfg.ModuleRoot), sub) @@ -392,12 +405,15 @@ func (l *loader) absDirFromImportPath(pos token.Pos, p importPath) (absDir, name } func (l *loader) externalPackageDir(p importPath) (dir string, err error) { - m, subPath, ok := l.deps.lookup(p) - if !ok { - return "", fmt.Errorf("no dependency found for import path %q", p) + if l.deps == nil { + return "", fmt.Errorf("no dependency found for import path %q (no dependencies at all)", p) + } + m, subPath, err := l.deps.lookup(p) + if err != nil { + return "", err } - dir, err = l.regClient.getModContents(m) + dir, err = l.regClient.getModContents(context.TODO(), m) if err != nil { return "", fmt.Errorf("cannot get contents for %v: %v", m, err) } diff --git a/cue/load/import_test.go b/cue/load/import_test.go index 8ca804d9741..479a5b1f638 100644 --- a/cue/load/import_test.go +++ b/cue/load/import_test.go @@ -15,6 +15,7 @@ package load import ( + "fmt" "os" "path/filepath" "reflect" @@ -37,7 +38,10 @@ func getInst(pkg, cwd string) (*build.Instance, error) { // all the way to the root of the git repository, causing Go's test caching // to never kick in, as the .git directory almost always changes. // Moreover, it's extra work that isn't useful to the tests. - c, _ := (&Config{ModuleRoot: cwd, Dir: cwd}).complete() + c, err := (&Config{ModuleRoot: cwd, Dir: cwd}).complete() + if err != nil { + return nil, fmt.Errorf("unexpected error on Config.complete: %v", err) + } l := loader{cfg: c} inst := l.newRelInstance(token.NoPos, pkg, c.Package) p := l.importPkg(token.NoPos, inst)[0] @@ -45,7 +49,12 @@ func getInst(pkg, cwd string) (*build.Instance, error) { } func TestEmptyImport(t *testing.T) { - c, _ := (&Config{}).complete() + c, err := (&Config{ + ModuleRoot: ".", + }).complete() + if err != nil { + t.Fatal(err) + } l := loader{cfg: c} inst := l.newInstance(token.NoPos, "") p := l.importPkg(token.NoPos, inst)[0] diff --git a/cue/load/instances.go b/cue/load/instances.go index f4c0de918a6..becd322979a 100644 --- a/cue/load/instances.go +++ b/cue/load/instances.go @@ -20,6 +20,7 @@ package load // - go/build import ( + "fmt" "os" "cuelang.org/go/cue/ast" @@ -56,12 +57,16 @@ func Instances(args []string, c *Config) []*build.Instance { if err != nil { return []*build.Instance{c.newErrInstance(err)} } - regClient = newRegistryClient(c.Registry, tmpDir) + regClient, err = newRegistryClient(c.Registry, tmpDir) + if err != nil { + return []*build.Instance{c.newErrInstance(fmt.Errorf("cannot make registry client: %v", err))} + } deps1, err := resolveDependencies(c.modFile, regClient) if err != nil { return []*build.Instance{c.newErrInstance(err)} } deps = deps1 + } tg := newTagger(c) l := newLoader(c, tg, deps, regClient) diff --git a/cue/load/internal/mvs/errors.go b/cue/load/internal/mvs/errors.go deleted file mode 100644 index bf183cea9e8..00000000000 --- a/cue/load/internal/mvs/errors.go +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright 2020 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package mvs - -import ( - "fmt" - "strings" - - "golang.org/x/mod/module" -) - -// BuildListError decorates an error that occurred gathering requirements -// while constructing a build list. BuildListError prints the chain -// of requirements to the module where the error occurred. -type BuildListError struct { - Err error - stack []buildListErrorElem -} - -type buildListErrorElem struct { - m module.Version - - // nextReason is the reason this module depends on the next module in the - // stack. Typically either "requires", or "updating to". - nextReason string -} - -// NewBuildListError returns a new BuildListError wrapping an error that -// occurred at a module found along the given path of requirements and/or -// upgrades, which must be non-empty. -// -// The isVersionChange function reports whether a path step is due to an -// explicit upgrade or downgrade (as opposed to an existing requirement in a -// go.mod file). A nil isVersionChange function indicates that none of the path -// steps are due to explicit version changes. -func NewBuildListError(err error, path []module.Version, isVersionChange func(from, to module.Version) bool) *BuildListError { - stack := make([]buildListErrorElem, 0, len(path)) - for len(path) > 1 { - reason := "requires" - if isVersionChange != nil && isVersionChange(path[0], path[1]) { - reason = "updating to" - } - stack = append(stack, buildListErrorElem{ - m: path[0], - nextReason: reason, - }) - path = path[1:] - } - stack = append(stack, buildListErrorElem{m: path[0]}) - - return &BuildListError{ - Err: err, - stack: stack, - } -} - -// Module returns the module where the error occurred. If the module stack -// is empty, this returns a zero value. -func (e *BuildListError) Module() module.Version { - if len(e.stack) == 0 { - return module.Version{} - } - return e.stack[len(e.stack)-1].m -} - -func (e *BuildListError) Error() string { - b := &strings.Builder{} - stack := e.stack - - // Don't print modules at the beginning of the chain without a - // version. These always seem to be the main module or a - // synthetic module ("target@"). - for len(stack) > 0 && stack[0].m.Version == "" { - stack = stack[1:] - } - - if len(stack) == 0 { - b.WriteString(e.Err.Error()) - } else { - for _, elem := range stack[:len(stack)-1] { - fmt.Fprintf(b, "%s %s\n\t", elem.m, elem.nextReason) - } - // Ensure that the final module path and version are included as part of the - // error message. - m := stack[len(stack)-1].m - if mErr, ok := e.Err.(*module.ModuleError); ok { - actual := module.Version{Path: mErr.Path, Version: mErr.Version} - if v, ok := mErr.Err.(*module.InvalidVersionError); ok { - actual.Version = v.Version - } - if actual == m { - fmt.Fprintf(b, "%v", e.Err) - } else { - fmt.Fprintf(b, "%s (replaced by %s): %v", m, actual, mErr.Err) - } - } else { - fmt.Fprintf(b, "%v", module.VersionError(m, e.Err)) - } - } - return b.String() -} diff --git a/cue/load/internal/mvs/graph.go b/cue/load/internal/mvs/graph.go deleted file mode 100644 index 364a5b1d85e..00000000000 --- a/cue/load/internal/mvs/graph.go +++ /dev/null @@ -1,225 +0,0 @@ -// Copyright 2020 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package mvs - -import ( - "fmt" - - "cuelang.org/go/cue/load/internal/slices" - - "golang.org/x/mod/module" -) - -// Graph implements an incremental version of the MVS algorithm, with the -// requirements pushed by the caller instead of pulled by the MVS traversal. -type Graph struct { - cmp func(v1, v2 string) int - roots []module.Version - - required map[module.Version][]module.Version - - isRoot map[module.Version]bool // contains true for roots and false for reachable non-roots - selected map[string]string // path → version -} - -// NewGraph returns an incremental MVS graph containing only a set of root -// dependencies and using the given max function for version strings. -// -// The caller must ensure that the root slice is not modified while the Graph -// may be in use. -func NewGraph(cmp func(v1, v2 string) int, roots []module.Version) *Graph { - g := &Graph{ - cmp: cmp, - roots: slices.Clip(roots), - required: make(map[module.Version][]module.Version), - isRoot: make(map[module.Version]bool), - selected: make(map[string]string), - } - - for _, m := range roots { - g.isRoot[m] = true - if g.cmp(g.Selected(m.Path), m.Version) < 0 { - g.selected[m.Path] = m.Version - } - } - - return g -} - -// Require adds the information that module m requires all modules in reqs. -// The reqs slice must not be modified after it is passed to Require. -// -// m must be reachable by some existing chain of requirements from g's target, -// and Require must not have been called for it already. -// -// If any of the modules in reqs has the same path as g's target, -// the target must have higher precedence than the version in req. -func (g *Graph) Require(m module.Version, reqs []module.Version) { - // To help catch disconnected-graph bugs, enforce that all required versions - // are actually reachable from the roots (and therefore should affect the - // selected versions of the modules they name). - if _, reachable := g.isRoot[m]; !reachable { - panic(fmt.Sprintf("%v is not reachable from any root", m)) - } - - // Truncate reqs to its capacity to avoid aliasing bugs if it is later - // returned from RequiredBy and appended to. - reqs = slices.Clip(reqs) - - if _, dup := g.required[m]; dup { - panic(fmt.Sprintf("requirements of %v have already been set", m)) - } - g.required[m] = reqs - - for _, dep := range reqs { - // Mark dep reachable, regardless of whether it is selected. - if _, ok := g.isRoot[dep]; !ok { - g.isRoot[dep] = false - } - - if g.cmp(g.Selected(dep.Path), dep.Version) < 0 { - g.selected[dep.Path] = dep.Version - } - } -} - -// RequiredBy returns the slice of requirements passed to Require for m, if any, -// with its capacity reduced to its length. -// If Require has not been called for m, RequiredBy(m) returns ok=false. -// -// The caller must not modify the returned slice, but may safely append to it -// and may rely on it not to be modified. -func (g *Graph) RequiredBy(m module.Version) (reqs []module.Version, ok bool) { - reqs, ok = g.required[m] - return reqs, ok -} - -// Selected returns the selected version of the given module path. -// -// If no version is selected, Selected returns version "none". -func (g *Graph) Selected(path string) (version string) { - v, ok := g.selected[path] - if !ok { - return "none" - } - return v -} - -// BuildList returns the selected versions of all modules present in the Graph, -// beginning with the selected versions of each module path in the roots of g. -// -// The order of the remaining elements in the list is deterministic -// but arbitrary. -func (g *Graph) BuildList() []module.Version { - seenRoot := make(map[string]bool, len(g.roots)) - - var list []module.Version - for _, r := range g.roots { - if seenRoot[r.Path] { - // Multiple copies of the same root, with the same or different versions, - // are a bit of a degenerate case: we will take the transitive - // requirements of both roots into account, but only the higher one can - // possibly be selected. However — especially given that we need the - // seenRoot map for later anyway — it is simpler to support this - // degenerate case than to forbid it. - continue - } - - if v := g.Selected(r.Path); v != "none" { - list = append(list, module.Version{Path: r.Path, Version: v}) - } - seenRoot[r.Path] = true - } - uniqueRoots := list - - for path, version := range g.selected { - if !seenRoot[path] { - list = append(list, module.Version{Path: path, Version: version}) - } - } - module.Sort(list[len(uniqueRoots):]) - - return list -} - -// WalkBreadthFirst invokes f once, in breadth-first order, for each module -// version other than "none" that appears in the graph, regardless of whether -// that version is selected. -func (g *Graph) WalkBreadthFirst(f func(m module.Version)) { - var queue []module.Version - enqueued := make(map[module.Version]bool) - for _, m := range g.roots { - if m.Version != "none" { - queue = append(queue, m) - enqueued[m] = true - } - } - - for len(queue) > 0 { - m := queue[0] - queue = queue[1:] - - f(m) - - reqs, _ := g.RequiredBy(m) - for _, r := range reqs { - if !enqueued[r] && r.Version != "none" { - queue = append(queue, r) - enqueued[r] = true - } - } - } -} - -// FindPath reports a shortest requirement path starting at one of the roots of -// the graph and ending at a module version m for which f(m) returns true, or -// nil if no such path exists. -func (g *Graph) FindPath(f func(module.Version) bool) []module.Version { - // firstRequires[a] = b means that in a breadth-first traversal of the - // requirement graph, the module version a was first required by b. - firstRequires := make(map[module.Version]module.Version) - - queue := g.roots - for _, m := range g.roots { - firstRequires[m] = module.Version{} - } - - for len(queue) > 0 { - m := queue[0] - queue = queue[1:] - - if f(m) { - // Construct the path reversed (because we're starting from the far - // endpoint), then reverse it. - path := []module.Version{m} - for { - m = firstRequires[m] - if m.Path == "" { - break - } - path = append(path, m) - } - - i, j := 0, len(path)-1 - for i < j { - path[i], path[j] = path[j], path[i] - i++ - j-- - } - - return path - } - - reqs, _ := g.RequiredBy(m) - for _, r := range reqs { - if _, seen := firstRequires[r]; !seen { - queue = append(queue, r) - firstRequires[r] = m - } - } - } - - return nil -} diff --git a/cue/load/internal/mvs/mvs.go b/cue/load/internal/mvs/mvs.go deleted file mode 100644 index cdc04a2fb2c..00000000000 --- a/cue/load/internal/mvs/mvs.go +++ /dev/null @@ -1,487 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package mvs implements Minimal Version Selection. -// See https://research.swtch.com/vgo-mvs. -package mvs - -import ( - "fmt" - "reflect" - "sort" - "sync" - - "cuelang.org/go/cue/load/internal/par" - - "golang.org/x/mod/module" -) - -// A Reqs is the requirement graph on which Minimal Version Selection (MVS) operates. -// -// The version strings are opaque except for the special version "none" -// (see the documentation for module.Version). In particular, MVS does not -// assume that the version strings are semantic versions; instead, the Max method -// gives access to the comparison operation. -// -// It must be safe to call methods on a Reqs from multiple goroutines simultaneously. -// Because a Reqs may read the underlying graph from the network on demand, -// the MVS algorithms parallelize the traversal to overlap network delays. -type Reqs interface { - // Required returns the module versions explicitly required by m itself. - // The caller must not modify the returned list. - Required(m module.Version) ([]module.Version, error) - - // Max returns the maximum of v1 and v2 (it returns either v1 or v2). - // - // For all versions v, Max(v, "none") must be v, - // and for the target passed as the first argument to MVS functions, - // Max(target, v) must be target. - // - // Note that v1 < v2 can be written Max(v1, v2) != v1 - // and similarly v1 <= v2 can be written Max(v1, v2) == v2. - Max(v1, v2 string) string -} - -// An UpgradeReqs is a Reqs that can also identify available upgrades. -type UpgradeReqs interface { - Reqs - - // Upgrade returns the upgraded version of m, - // for use during an UpgradeAll operation. - // If m should be kept as is, Upgrade returns m. - // If m is not yet used in the build, then m.Version will be "none". - // More typically, m.Version will be the version required - // by some other module in the build. - // - // If no module version is available for the given path, - // Upgrade returns a non-nil error. - // TODO(rsc): Upgrade must be able to return errors, - // but should "no latest version" just return m instead? - Upgrade(m module.Version) (module.Version, error) -} - -// A DowngradeReqs is a Reqs that can also identify available downgrades. -type DowngradeReqs interface { - Reqs - - // Previous returns the version of m.Path immediately prior to m.Version, - // or "none" if no such version is known. - Previous(m module.Version) (module.Version, error) -} - -// BuildList returns the build list for the target module. -// -// target is the root vertex of a module requirement graph. For cmd/go, this is -// typically the main module, but note that this algorithm is not intended to -// be Go-specific: module paths and versions are treated as opaque values. -// -// reqs describes the module requirement graph and provides an opaque method -// for comparing versions. -// -// BuildList traverses the graph and returns a list containing the highest -// version for each visited module. The first element of the returned list is -// target itself; reqs.Max requires target.Version to compare higher than all -// other versions, so no other version can be selected. The remaining elements -// of the list are sorted by path. -// -// See https://research.swtch.com/vgo-mvs for details. -func BuildList(targets []module.Version, reqs Reqs) ([]module.Version, error) { - return buildList(targets, reqs, nil) -} - -func buildList(targets []module.Version, reqs Reqs, upgrade func(module.Version) (module.Version, error)) ([]module.Version, error) { - cmp := func(v1, v2 string) int { - if reqs.Max(v1, v2) != v1 { - return -1 - } - if reqs.Max(v2, v1) != v2 { - return 1 - } - return 0 - } - - var ( - mu sync.Mutex - g = NewGraph(cmp, targets) - upgrades = map[module.Version]module.Version{} - errs = map[module.Version]error{} // (non-nil errors only) - ) - - // Explore work graph in parallel in case reqs.Required - // does high-latency network operations. - var work par.Work[module.Version] - for _, target := range targets { - work.Add(target) - } - work.Do(10, func(m module.Version) { - - var required []module.Version - var err error - if m.Version != "none" { - required, err = reqs.Required(m) - } - - u := m - if upgrade != nil { - upgradeTo, upErr := upgrade(m) - if upErr == nil { - u = upgradeTo - } else if err == nil { - err = upErr - } - } - - mu.Lock() - if err != nil { - errs[m] = err - } - if u != m { - upgrades[m] = u - required = append([]module.Version{u}, required...) - } - g.Require(m, required) - mu.Unlock() - - for _, r := range required { - work.Add(r) - } - }) - - // If there was an error, find the shortest path from the target to the - // node where the error occurred so we can report a useful error message. - if len(errs) > 0 { - errPath := g.FindPath(func(m module.Version) bool { - return errs[m] != nil - }) - if len(errPath) == 0 { - panic("internal error: could not reconstruct path to module with error") - } - - err := errs[errPath[len(errPath)-1]] - isUpgrade := func(from, to module.Version) bool { - if u, ok := upgrades[from]; ok { - return u == to - } - return false - } - return nil, NewBuildListError(err, errPath, isUpgrade) - } - - // The final list is the minimum version of each module found in the graph. - list := g.BuildList() - if vs := list[:len(targets)]; !reflect.DeepEqual(vs, targets) { - // target.Version will be "" for modload, the main client of MVS. - // "" denotes the main module, which has no version. However, MVS treats - // version strings as opaque, so "" is not a special value here. - // See golang.org/issue/31491, golang.org/issue/29773. - panic(fmt.Sprintf("mistake: chose versions %+v instead of targets %+v", vs, targets)) - } - return list, nil -} - -// Req returns the minimal requirement list for the target module, -// with the constraint that all module paths listed in base must -// appear in the returned list. -func Req(mainModule module.Version, base []string, reqs Reqs) ([]module.Version, error) { - list, err := BuildList([]module.Version{mainModule}, reqs) - if err != nil { - return nil, err - } - - // Note: Not running in parallel because we assume - // that list came from a previous operation that paged - // in all the requirements, so there's no I/O to overlap now. - - max := map[string]string{} - for _, m := range list { - max[m.Path] = m.Version - } - - // Compute postorder, cache requirements. - var postorder []module.Version - reqCache := map[module.Version][]module.Version{} - reqCache[mainModule] = nil - - var walk func(module.Version) error - walk = func(m module.Version) error { - _, ok := reqCache[m] - if ok { - return nil - } - required, err := reqs.Required(m) - if err != nil { - return err - } - reqCache[m] = required - for _, m1 := range required { - if err := walk(m1); err != nil { - return err - } - } - postorder = append(postorder, m) - return nil - } - for _, m := range list { - if err := walk(m); err != nil { - return nil, err - } - } - - // Walk modules in reverse post-order, only adding those not implied already. - have := map[module.Version]bool{} - walk = func(m module.Version) error { - if have[m] { - return nil - } - have[m] = true - for _, m1 := range reqCache[m] { - walk(m1) - } - return nil - } - // First walk the base modules that must be listed. - var min []module.Version - haveBase := map[string]bool{} - for _, path := range base { - if haveBase[path] { - continue - } - m := module.Version{Path: path, Version: max[path]} - min = append(min, m) - walk(m) - haveBase[path] = true - } - // Now the reverse postorder to bring in anything else. - for i := len(postorder) - 1; i >= 0; i-- { - m := postorder[i] - if max[m.Path] != m.Version { - // Older version. - continue - } - if !have[m] { - min = append(min, m) - walk(m) - } - } - sort.Slice(min, func(i, j int) bool { - return min[i].Path < min[j].Path - }) - return min, nil -} - -// UpgradeAll returns a build list for the target module -// in which every module is upgraded to its latest version. -func UpgradeAll(target module.Version, reqs UpgradeReqs) ([]module.Version, error) { - return buildList([]module.Version{target}, reqs, func(m module.Version) (module.Version, error) { - if m.Path == target.Path { - return target, nil - } - - return reqs.Upgrade(m) - }) -} - -// Upgrade returns a build list for the target module -// in which the given additional modules are upgraded. -func Upgrade(target module.Version, reqs UpgradeReqs, upgrade ...module.Version) ([]module.Version, error) { - list, err := reqs.Required(target) - if err != nil { - return nil, err - } - - pathInList := make(map[string]bool, len(list)) - for _, m := range list { - pathInList[m.Path] = true - } - list = append([]module.Version(nil), list...) - - upgradeTo := make(map[string]string, len(upgrade)) - for _, u := range upgrade { - if !pathInList[u.Path] { - list = append(list, module.Version{Path: u.Path, Version: "none"}) - } - if prev, dup := upgradeTo[u.Path]; dup { - upgradeTo[u.Path] = reqs.Max(prev, u.Version) - } else { - upgradeTo[u.Path] = u.Version - } - } - - return buildList([]module.Version{target}, &override{target, list, reqs}, func(m module.Version) (module.Version, error) { - if v, ok := upgradeTo[m.Path]; ok { - return module.Version{Path: m.Path, Version: v}, nil - } - return m, nil - }) -} - -// Downgrade returns a build list for the target module -// in which the given additional modules are downgraded, -// potentially overriding the requirements of the target. -// -// The versions to be downgraded may be unreachable from reqs.Latest and -// reqs.Previous, but the methods of reqs must otherwise handle such versions -// correctly. -func Downgrade(target module.Version, reqs DowngradeReqs, downgrade ...module.Version) ([]module.Version, error) { - // Per https://research.swtch.com/vgo-mvs#algorithm_4: - // “To avoid an unnecessary downgrade to E 1.1, we must also add a new - // requirement on E 1.2. We can apply Algorithm R to find the minimal set of - // new requirements to write to go.mod.” - // - // In order to generate those new requirements, we need to identify versions - // for every module in the build list — not just reqs.Required(target). - list, err := BuildList([]module.Version{target}, reqs) - if err != nil { - return nil, err - } - list = list[1:] // remove target - - max := make(map[string]string) - for _, r := range list { - max[r.Path] = r.Version - } - for _, d := range downgrade { - if v, ok := max[d.Path]; !ok || reqs.Max(v, d.Version) != d.Version { - max[d.Path] = d.Version - } - } - - var ( - added = make(map[module.Version]bool) - rdeps = make(map[module.Version][]module.Version) - excluded = make(map[module.Version]bool) - ) - var exclude func(module.Version) - exclude = func(m module.Version) { - if excluded[m] { - return - } - excluded[m] = true - for _, p := range rdeps[m] { - exclude(p) - } - } - var add func(module.Version) - add = func(m module.Version) { - if added[m] { - return - } - added[m] = true - if v, ok := max[m.Path]; ok && reqs.Max(m.Version, v) != v { - // m would upgrade an existing dependency — it is not a strict downgrade, - // and because it was already present as a dependency, it could affect the - // behavior of other relevant packages. - exclude(m) - return - } - list, err := reqs.Required(m) - if err != nil { - // If we can't load the requirements, we couldn't load the go.mod file. - // There are a number of reasons this can happen, but this usually - // means an older version of the module had a missing or invalid - // go.mod file. For example, if example.com/mod released v2.0.0 before - // migrating to modules (v2.0.0+incompatible), then added a valid go.mod - // in v2.0.1, downgrading from v2.0.1 would cause this error. - // - // TODO(golang.org/issue/31730, golang.org/issue/30134): if the error - // is transient (we couldn't download go.mod), return the error from - // Downgrade. Currently, we can't tell what kind of error it is. - exclude(m) - return - } - for _, r := range list { - add(r) - if excluded[r] { - exclude(m) - return - } - rdeps[r] = append(rdeps[r], m) - } - } - - downgraded := make([]module.Version, 0, len(list)+1) - downgraded = append(downgraded, target) -List: - for _, r := range list { - add(r) - for excluded[r] { - p, err := reqs.Previous(r) - if err != nil { - // This is likely a transient error reaching the repository, - // rather than a permanent error with the retrieved version. - // - // TODO(golang.org/issue/31730, golang.org/issue/30134): - // decode what to do based on the actual error. - return nil, err - } - // If the target version is a pseudo-version, it may not be - // included when iterating over prior versions using reqs.Previous. - // Insert it into the right place in the iteration. - // If v is excluded, p should be returned again by reqs.Previous on the next iteration. - if v := max[r.Path]; reqs.Max(v, r.Version) != v && reqs.Max(p.Version, v) != p.Version { - p.Version = v - } - if p.Version == "none" { - continue List - } - add(p) - r = p - } - downgraded = append(downgraded, r) - } - - // The downgrades we computed above only downgrade to versions enumerated by - // reqs.Previous. However, reqs.Previous omits some versions — such as - // pseudo-versions and retracted versions — that may be selected as transitive - // requirements of other modules. - // - // If one of those requirements pulls the version back up above the version - // identified by reqs.Previous, then the transitive dependencies of that - // initially-downgraded version should no longer matter — in particular, we - // should not add new dependencies on module paths that nothing else in the - // updated module graph even requires. - // - // In order to eliminate those spurious dependencies, we recompute the build - // list with the actual versions of the downgraded modules as selected by MVS, - // instead of our initial downgrades. - // (See the downhiddenartifact and downhiddencross test cases). - actual, err := BuildList([]module.Version{target}, &override{ - target: target, - list: downgraded, - Reqs: reqs, - }) - if err != nil { - return nil, err - } - actualVersion := make(map[string]string, len(actual)) - for _, m := range actual { - actualVersion[m.Path] = m.Version - } - - downgraded = downgraded[:0] - for _, m := range list { - if v, ok := actualVersion[m.Path]; ok { - downgraded = append(downgraded, module.Version{Path: m.Path, Version: v}) - } - } - - return BuildList([]module.Version{target}, &override{ - target: target, - list: downgraded, - Reqs: reqs, - }) -} - -type override struct { - target module.Version - list []module.Version - Reqs -} - -func (r *override) Required(m module.Version) ([]module.Version, error) { - if m == r.target { - return r.list, nil - } - return r.Reqs.Required(m) -} diff --git a/cue/load/internal/mvs/mvs_test.go b/cue/load/internal/mvs/mvs_test.go deleted file mode 100644 index 26d004fee28..00000000000 --- a/cue/load/internal/mvs/mvs_test.go +++ /dev/null @@ -1,635 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package mvs - -import ( - "fmt" - "reflect" - "strings" - "testing" - - "golang.org/x/mod/module" -) - -var tests = ` -# Scenario from blog. -name: blog -A: B1 C2 -B1: D3 -C1: D2 -C2: D4 -C3: D5 -C4: G1 -D2: E1 -D3: E2 -D4: E2 F1 -D5: E2 -G1: C4 -A2: B1 C4 D4 -build A: A B1 C2 D4 E2 F1 -upgrade* A: A B1 C4 D5 E2 F1 G1 -upgrade A C4: A B1 C4 D4 E2 F1 G1 -build A2: A2 B1 C4 D4 E2 F1 G1 -downgrade A2 D2: A2 C4 D2 E2 F1 G1 - -name: trim -A: B1 C2 -B1: D3 -C2: B2 -B2: -build A: A B2 C2 D3 - -# Cross-dependency between D and E. -# No matter how it arises, should get result of merging all build lists via max, -# which leads to including both D2 and E2. - -name: cross1 -A: B C -B: D1 -C: D2 -D1: E2 -D2: E1 -build A: A B C D2 E2 - -name: cross1V -A: B2 C D2 E1 -B1: -B2: D1 -C: D2 -D1: E2 -D2: E1 -build A: A B2 C D2 E2 - -name: cross1U -A: B1 C -B1: -B2: D1 -C: D2 -D1: E2 -D2: E1 -build A: A B1 C D2 E1 -upgrade A B2: A B2 C D2 E2 - -name: cross1R -A: B C -B: D2 -C: D1 -D1: E2 -D2: E1 -build A: A B C D2 E2 - -name: cross1X -A: B C -B: D1 E2 -C: D2 -D1: E2 -D2: E1 -build A: A B C D2 E2 - -name: cross2 -A: B D2 -B: D1 -D1: E2 -D2: E1 -build A: A B D2 E2 - -name: cross2X -A: B D2 -B: D1 E2 -C: D2 -D1: E2 -D2: E1 -build A: A B D2 E2 - -name: cross3 -A: B D2 E1 -B: D1 -D1: E2 -D2: E1 -build A: A B D2 E2 - -name: cross3X -A: B D2 E1 -B: D1 E2 -D1: E2 -D2: E1 -build A: A B D2 E2 - -# Should not get E2 here, because B has been updated -# not to depend on D1 anymore. -name: cross4 -A1: B1 D2 -A2: B2 D2 -B1: D1 -B2: D2 -D1: E2 -D2: E1 -build A1: A1 B1 D2 E2 -build A2: A2 B2 D2 E1 - -# But the upgrade from A1 preserves the E2 dep explicitly. -upgrade A1 B2: A1 B2 D2 E2 -upgradereq A1 B2: B2 E2 - -name: cross5 -A: D1 -D1: E2 -D2: E1 -build A: A D1 E2 -upgrade* A: A D2 E2 -upgrade A D2: A D2 E2 -upgradereq A D2: D2 E2 - -name: cross6 -A: D2 -D1: E2 -D2: E1 -build A: A D2 E1 -upgrade* A: A D2 E2 -upgrade A E2: A D2 E2 - -name: cross7 -A: B C -B: D1 -C: E1 -D1: E2 -E1: D2 -build A: A B C D2 E2 - -# golang.org/issue/31248: -# Even though we select X2, the requirement on I1 -# via X1 should be preserved. -name: cross8 -M: A1 B1 -A1: X1 -B1: X2 -X1: I1 -X2: -build M: M A1 B1 I1 X2 - -# Upgrade from B1 to B2 should not drop the transitive dep on D. -name: drop -A: B1 C1 -B1: D1 -B2: -C2: -D2: -build A: A B1 C1 D1 -upgrade* A: A B2 C2 D2 - -name: simplify -A: B1 C1 -B1: C2 -C1: D1 -C2: -build A: A B1 C2 D1 - -name: up1 -A: B1 C1 -B1: -B2: -B3: -B4: -B5.hidden: -C2: -C3: -build A: A B1 C1 -upgrade* A: A B4 C3 - -name: up2 -A: B5.hidden C1 -B1: -B2: -B3: -B4: -B5.hidden: -C2: -C3: -build A: A B5.hidden C1 -upgrade* A: A B5.hidden C3 - -name: down1 -A: B2 -B1: C1 -B2: C2 -build A: A B2 C2 -downgrade A C1: A B1 C1 - -name: down2 -A: B2 E2 -B1: -B2: C2 F2 -C1: -D1: -C2: D2 E2 -D2: B2 -E2: D2 -E1: -F1: -build A: A B2 C2 D2 E2 F2 -downgrade A F1: A B1 C1 D1 E1 F1 - -# https://research.swtch.com/vgo-mvs#algorithm_4: -# “[D]owngrades are constrained to only downgrade packages, not also upgrade -# them; if an upgrade before downgrade is needed, the user must ask for it -# explicitly.” -# -# Here, downgrading B2 to B1 upgrades C1 to C2, and C2 does not depend on D2. -# However, C2 would be an upgrade — not a downgrade — so B1 must also be -# rejected. -name: downcross1 -A: B2 C1 -B1: C2 -B2: C1 -C1: D2 -C2: -D1: -D2: -build A: A B2 C1 D2 -downgrade A D1: A D1 - -# https://research.swtch.com/vgo-mvs#algorithm_4: -# “Unlike upgrades, downgrades must work by removing requirements, not adding -# them.” -# -# However, downgrading a requirement may introduce a new requirement on a -# previously-unrequired module. If each dependency's requirements are complete -# (“tidy”), that can't change the behavior of any other package whose version is -# not also being downgraded, so we should allow it. -name: downcross2 -A: B2 -B1: C1 -B2: D2 -C1: -D1: -D2: -build A: A B2 D2 -downgrade A D1: A B1 C1 D1 - -name: downcycle -A: A B2 -B2: A -B1: -build A: A B2 -downgrade A B1: A B1 - -# Both B3 and C2 require D2. -# If we downgrade D to D1, then in isolation B3 would downgrade to B1, -# because B2 is hidden — B1 is the next-highest version that is not hidden. -# However, if we downgrade D, we will also downgrade C to C1. -# And C1 requires B2.hidden, and B2.hidden also meets our requirements: -# it is compatible with D1 and a strict downgrade from B3. -# -# Since neither the initial nor the final build list includes B1, -# and the nothing in the final downgraded build list requires E at all, -# no dependency on E1 (required by only B1) should be introduced. -# -name: downhiddenartifact -A: B3 C2 -A1: B3 -B1: E1 -B2.hidden: -B3: D2 -C1: B2.hidden -C2: D2 -D1: -D2: -build A1: A1 B3 D2 -downgrade A1 D1: A1 B1 D1 E1 -build A: A B3 C2 D2 -downgrade A D1: A B2.hidden C1 D1 - -# Both B3 and C3 require D2. -# If we downgrade D to D1, then in isolation B3 would downgrade to B1, -# and C3 would downgrade to C1. -# But C1 requires B2.hidden, and B1 requires C2.hidden, so we can't -# downgrade to either of those without pulling the other back up a little. -# -# B2.hidden and C2.hidden are both compatible with D1, so that still -# meets our requirements — but then we're in an odd state in which -# B and C have both been downgraded to hidden versions, without any -# remaining requirements to explain how those hidden versions got there. -# -# TODO(bcmills): Would it be better to force downgrades to land on non-hidden -# versions? -# In this case, that would remove the dependencies on B and C entirely. -# -name: downhiddencross -A: B3 C3 -B1: C2.hidden -B2.hidden: -B3: D2 -C1: B2.hidden -C2.hidden: -C3: D2 -D1: -D2: -build A: A B3 C3 D2 -downgrade A D1: A B2.hidden C2.hidden D1 - -# golang.org/issue/25542. -name: noprev1 -A: B4 C2 -B2.hidden: -C2: -build A: A B4 C2 -downgrade A B2.hidden: A B2.hidden C2 - -name: noprev2 -A: B4 C2 -B2.hidden: -B1: -C2: -build A: A B4 C2 -downgrade A B2.hidden: A B2.hidden C2 - -name: noprev3 -A: B4 C2 -B3: -B2.hidden: -C2: -build A: A B4 C2 -downgrade A B2.hidden: A B2.hidden C2 - -# Cycles involving the target. - -# The target must be the newest version of itself. -name: cycle1 -A: B1 -B1: A1 -B2: A2 -B3: A3 -build A: A B1 -upgrade A B2: A B2 -upgrade* A: A B3 - -# golang.org/issue/29773: -# Requirements of older versions of the target -# must be carried over. -name: cycle2 -A: B1 -A1: C1 -A2: D1 -B1: A1 -B2: A2 -C1: A2 -C2: -D2: -build A: A B1 C1 D1 -upgrade* A: A B2 C2 D2 - -# Cycles with multiple possible solutions. -# (golang.org/issue/34086) -name: cycle3 -M: A1 C2 -A1: B1 -B1: C1 -B2: C2 -C1: -C2: B2 -build M: M A1 B2 C2 -req M: A1 B2 -req M A: A1 B2 -req M C: A1 C2 - -# Requirement minimization. - -name: req1 -A: B1 C1 D1 E1 F1 -B1: C1 E1 F1 -req A: B1 D1 -req A C: B1 C1 D1 - -name: req2 -A: G1 H1 -G1: H1 -H1: G1 -req A: G1 -req A G: G1 -req A H: H1 - -name: req3 -M: A1 B1 -A1: X1 -B1: X2 -X1: I1 -X2: -req M: A1 B1 - -name: reqnone -M: Anone B1 D1 E1 -B1: Cnone D1 -E1: Fnone -build M: M B1 D1 E1 -req M: B1 E1 - -name: reqdup -M: A1 B1 -A1: B1 -B1: -req M A A: A1 - -name: reqcross -M: A1 B1 C1 -A1: B1 C1 -B1: C1 -C1: -req M A B: A1 B1 -` - -func Test(t *testing.T) { - var ( - name string - reqs reqsMap - fns []func(*testing.T) - ) - flush := func() { - if name != "" { - t.Run(name, func(t *testing.T) { - for _, fn := range fns { - fn(t) - } - if len(fns) == 0 { - t.Errorf("no functions tested") - } - }) - } - } - m := func(s string) module.Version { - return module.Version{Path: s[:1], Version: s[1:]} - } - ms := func(list []string) []module.Version { - var mlist []module.Version - for _, s := range list { - mlist = append(mlist, m(s)) - } - return mlist - } - checkList := func(t *testing.T, desc string, list []module.Version, err error, val string) { - if err != nil { - t.Fatalf("%s: %v", desc, err) - } - vs := ms(strings.Fields(val)) - if !reflect.DeepEqual(list, vs) { - t.Errorf("%s = %v, want %v", desc, list, vs) - } - } - - for _, line := range strings.Split(tests, "\n") { - line = strings.TrimSpace(line) - if strings.HasPrefix(line, "#") || line == "" { - continue - } - i := strings.Index(line, ":") - if i < 0 { - t.Fatalf("missing colon: %q", line) - } - key := strings.TrimSpace(line[:i]) - val := strings.TrimSpace(line[i+1:]) - if key == "" { - t.Fatalf("missing key: %q", line) - } - kf := strings.Fields(key) - switch kf[0] { - case "name": - if len(kf) != 1 { - t.Fatalf("name takes no arguments: %q", line) - } - flush() - reqs = make(reqsMap) - fns = nil - name = val - continue - case "build": - if len(kf) != 2 { - t.Fatalf("build takes one argument: %q", line) - } - fns = append(fns, func(t *testing.T) { - list, err := BuildList([]module.Version{m(kf[1])}, reqs) - checkList(t, key, list, err, val) - }) - continue - case "upgrade*": - if len(kf) != 2 { - t.Fatalf("upgrade* takes one argument: %q", line) - } - fns = append(fns, func(t *testing.T) { - list, err := UpgradeAll(m(kf[1]), reqs) - checkList(t, key, list, err, val) - }) - continue - case "upgradereq": - if len(kf) < 2 { - t.Fatalf("upgrade takes at least one argument: %q", line) - } - fns = append(fns, func(t *testing.T) { - list, err := Upgrade(m(kf[1]), reqs, ms(kf[2:])...) - if err == nil { - // Copy the reqs map, but substitute the upgraded requirements in - // place of the target's original requirements. - upReqs := make(reqsMap, len(reqs)) - for m, r := range reqs { - upReqs[m] = r - } - upReqs[m(kf[1])] = list - - list, err = Req(m(kf[1]), nil, upReqs) - } - checkList(t, key, list, err, val) - }) - continue - case "upgrade": - if len(kf) < 2 { - t.Fatalf("upgrade takes at least one argument: %q", line) - } - fns = append(fns, func(t *testing.T) { - list, err := Upgrade(m(kf[1]), reqs, ms(kf[2:])...) - checkList(t, key, list, err, val) - }) - continue - case "downgrade": - if len(kf) < 2 { - t.Fatalf("downgrade takes at least one argument: %q", line) - } - fns = append(fns, func(t *testing.T) { - list, err := Downgrade(m(kf[1]), reqs, ms(kf[1:])...) - checkList(t, key, list, err, val) - }) - continue - case "req": - if len(kf) < 2 { - t.Fatalf("req takes at least one argument: %q", line) - } - fns = append(fns, func(t *testing.T) { - list, err := Req(m(kf[1]), kf[2:], reqs) - checkList(t, key, list, err, val) - }) - continue - } - if len(kf) == 1 && 'A' <= key[0] && key[0] <= 'Z' { - var rs []module.Version - for _, f := range strings.Fields(val) { - r := m(f) - if reqs[r] == nil { - reqs[r] = []module.Version{} - } - rs = append(rs, r) - } - reqs[m(key)] = rs - continue - } - t.Fatalf("bad line: %q", line) - } - flush() -} - -type reqsMap map[module.Version][]module.Version - -func (r reqsMap) Max(v1, v2 string) string { - if v1 == "none" || v2 == "" { - return v2 - } - if v2 == "none" || v1 == "" { - return v1 - } - if v1 < v2 { - return v2 - } - return v1 -} - -func (r reqsMap) Upgrade(m module.Version) (module.Version, error) { - u := module.Version{Version: "none"} - for k := range r { - if k.Path == m.Path && r.Max(u.Version, k.Version) == k.Version && !strings.HasSuffix(k.Version, ".hidden") { - u = k - } - } - if u.Path == "" { - return module.Version{}, fmt.Errorf("missing module: %v", module.Version{Path: m.Path}) - } - return u, nil -} - -func (r reqsMap) Previous(m module.Version) (module.Version, error) { - var p module.Version - for k := range r { - if k.Path == m.Path && p.Version < k.Version && k.Version < m.Version && !strings.HasSuffix(k.Version, ".hidden") { - p = k - } - } - if p.Path == "" { - return module.Version{Path: m.Path, Version: "none"}, nil - } - return p, nil -} - -func (r reqsMap) Required(m module.Version) ([]module.Version, error) { - rr, ok := r[m] - if !ok { - return nil, fmt.Errorf("missing module: %v", m) - } - return rr, nil -} diff --git a/cue/load/internal/par/atomic_go1.18.go b/cue/load/internal/par/atomic_go1.18.go deleted file mode 100644 index 0bfc2a46808..00000000000 --- a/cue/load/internal/par/atomic_go1.18.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build !go1.19 - -package par - -import "sync/atomic" - -// atomicBool implements the atomic.Bool type for Go versions before go -// 1.19. It's a copy of the relevant parts of the Go 1.19 atomic.Bool -// code as of commit a4d5fbc3a48b63f19fcd2a4d040a85c75a2709b5. -type atomicBool struct { - v uint32 -} - -// Load atomically loads and returns the value stored in x. -func (x *atomicBool) Load() bool { return atomic.LoadUint32(&x.v) != 0 } - -// Store atomically stores val into x. -func (x *atomicBool) Store(val bool) { atomic.StoreUint32(&x.v, b32(val)) } - -// b32 returns a uint32 0 or 1 representing b. -func b32(b bool) uint32 { - if b { - return 1 - } - return 0 -} diff --git a/cue/load/internal/par/atomic_go1.19.go b/cue/load/internal/par/atomic_go1.19.go deleted file mode 100644 index 48fdb8aa5ed..00000000000 --- a/cue/load/internal/par/atomic_go1.19.go +++ /dev/null @@ -1,11 +0,0 @@ -//go:build go1.19 - -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package par - -import "sync/atomic" - -type atomicBool = atomic.Bool diff --git a/cue/load/internal/par/queue.go b/cue/load/internal/par/queue.go deleted file mode 100644 index 180bc75e343..00000000000 --- a/cue/load/internal/par/queue.go +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2020 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package par - -import "fmt" - -// Queue manages a set of work items to be executed in parallel. The number of -// active work items is limited, and excess items are queued sequentially. -type Queue struct { - maxActive int - st chan queueState -} - -type queueState struct { - active int // number of goroutines processing work; always nonzero when len(backlog) > 0 - backlog []func() - idle chan struct{} // if non-nil, closed when active becomes 0 -} - -// NewQueue returns a Queue that executes up to maxActive items in parallel. -// -// maxActive must be positive. -func NewQueue(maxActive int) *Queue { - if maxActive < 1 { - panic(fmt.Sprintf("par.NewQueue called with nonpositive limit (%d)", maxActive)) - } - - q := &Queue{ - maxActive: maxActive, - st: make(chan queueState, 1), - } - q.st <- queueState{} - return q -} - -// Add adds f as a work item in the queue. -// -// Add returns immediately, but the queue will be marked as non-idle until after -// f (and any subsequently-added work) has completed. -func (q *Queue) Add(f func()) { - st := <-q.st - if st.active == q.maxActive { - st.backlog = append(st.backlog, f) - q.st <- st - return - } - if st.active == 0 { - // Mark q as non-idle. - st.idle = nil - } - st.active++ - q.st <- st - - go func() { - for { - f() - - st := <-q.st - if len(st.backlog) == 0 { - if st.active--; st.active == 0 && st.idle != nil { - close(st.idle) - } - q.st <- st - return - } - f, st.backlog = st.backlog[0], st.backlog[1:] - q.st <- st - } - }() -} - -// Idle returns a channel that will be closed when q has no (active or enqueued) -// work outstanding. -func (q *Queue) Idle() <-chan struct{} { - st := <-q.st - defer func() { q.st <- st }() - - if st.idle == nil { - st.idle = make(chan struct{}) - if st.active == 0 { - close(st.idle) - } - } - - return st.idle -} diff --git a/cue/load/internal/par/queue_test.go b/cue/load/internal/par/queue_test.go deleted file mode 100644 index 1331e65f98a..00000000000 --- a/cue/load/internal/par/queue_test.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2020 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package par - -import ( - "sync" - "testing" -) - -func TestQueueIdle(t *testing.T) { - q := NewQueue(1) - select { - case <-q.Idle(): - default: - t.Errorf("NewQueue(1) is not initially idle.") - } - - started := make(chan struct{}) - unblock := make(chan struct{}) - q.Add(func() { - close(started) - <-unblock - }) - - <-started - idle := q.Idle() - select { - case <-idle: - t.Errorf("NewQueue(1) is marked idle while processing work.") - default: - } - - close(unblock) - <-idle // Should be closed as soon as the Add callback returns. -} - -func TestQueueBacklog(t *testing.T) { - const ( - maxActive = 2 - totalWork = 3 * maxActive - ) - - q := NewQueue(maxActive) - t.Logf("q = NewQueue(%d)", maxActive) - - var wg sync.WaitGroup - wg.Add(totalWork) - started := make([]chan struct{}, totalWork) - unblock := make(chan struct{}) - for i := range started { - started[i] = make(chan struct{}) - i := i - q.Add(func() { - close(started[i]) - <-unblock - wg.Done() - }) - } - - for i, c := range started { - if i < maxActive { - <-c // Work item i should be started immediately. - } else { - select { - case <-c: - t.Errorf("Work item %d started before previous items finished.", i) - default: - } - } - } - - close(unblock) - for _, c := range started[maxActive:] { - <-c - } - wg.Wait() -} diff --git a/cue/load/internal/par/work.go b/cue/load/internal/par/work.go deleted file mode 100644 index e36084c0ad8..00000000000 --- a/cue/load/internal/par/work.go +++ /dev/null @@ -1,222 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package par implements parallel execution helpers. -package par - -import ( - "errors" - "math/rand" - "sync" -) - -// Work manages a set of work items to be executed in parallel, at most once each. -// The items in the set must all be valid map keys. -type Work[T comparable] struct { - f func(T) // function to run for each item - running int // total number of runners - - mu sync.Mutex - added map[T]bool // items added to set - todo []T // items yet to be run - wait sync.Cond // wait when todo is empty - waiting int // number of runners waiting for todo -} - -func (w *Work[T]) init() { - if w.added == nil { - w.added = make(map[T]bool) - } -} - -// Add adds item to the work set, if it hasn't already been added. -func (w *Work[T]) Add(item T) { - w.mu.Lock() - w.init() - if !w.added[item] { - w.added[item] = true - w.todo = append(w.todo, item) - if w.waiting > 0 { - w.wait.Signal() - } - } - w.mu.Unlock() -} - -// Do runs f in parallel on items from the work set, -// with at most n invocations of f running at a time. -// It returns when everything added to the work set has been processed. -// At least one item should have been added to the work set -// before calling Do (or else Do returns immediately), -// but it is allowed for f(item) to add new items to the set. -// Do should only be used once on a given Work. -func (w *Work[T]) Do(n int, f func(item T)) { - if n < 1 { - panic("par.Work.Do: n < 1") - } - if w.running >= 1 { - panic("par.Work.Do: already called Do") - } - - w.running = n - w.f = f - w.wait.L = &w.mu - - for i := 0; i < n-1; i++ { - go w.runner() - } - w.runner() -} - -// runner executes work in w until both nothing is left to do -// and all the runners are waiting for work. -// (Then all the runners return.) -func (w *Work[T]) runner() { - for { - // Wait for something to do. - w.mu.Lock() - for len(w.todo) == 0 { - w.waiting++ - if w.waiting == w.running { - // All done. - w.wait.Broadcast() - w.mu.Unlock() - return - } - w.wait.Wait() - w.waiting-- - } - - // Pick something to do at random, - // to eliminate pathological contention - // in case items added at about the same time - // are most likely to contend. - i := rand.Intn(len(w.todo)) - item := w.todo[i] - w.todo[i] = w.todo[len(w.todo)-1] - w.todo = w.todo[:len(w.todo)-1] - w.mu.Unlock() - - w.f(item) - } -} - -// ErrCache is like Cache except that it also stores -// an error value alongside the cached value V. -type ErrCache[K comparable, V any] struct { - Cache[K, errValue[V]] -} - -type errValue[V any] struct { - v V - err error -} - -func (c *ErrCache[K, V]) Do(key K, f func() (V, error)) (V, error) { - v := c.Cache.Do(key, func() errValue[V] { - v, err := f() - return errValue[V]{v, err} - }) - return v.v, v.err -} - -var ErrCacheEntryNotFound = errors.New("cache entry not found") - -// Get returns the cached result associated with key. -// It returns ErrCacheEntryNotFound if there is no such result. -func (c *ErrCache[K, V]) Get(key K) (V, error) { - v, ok := c.Cache.Get(key) - if !ok { - v.err = ErrCacheEntryNotFound - } - return v.v, v.err -} - -// Cache runs an action once per key and caches the result. -type Cache[K comparable, V any] struct { - m sync.Map -} - -type cacheEntry[V any] struct { - done atomicBool - mu sync.Mutex - result V -} - -// Do calls the function f if and only if Do is being called for the first time with this key. -// No call to Do with a given key returns until the one call to f returns. -// Do returns the value returned by the one call to f. -func (c *Cache[K, V]) Do(key K, f func() V) V { - entryIface, ok := c.m.Load(key) - if !ok { - entryIface, _ = c.m.LoadOrStore(key, new(cacheEntry[V])) - } - e := entryIface.(*cacheEntry[V]) - if !e.done.Load() { - e.mu.Lock() - if !e.done.Load() { - e.result = f() - e.done.Store(true) - } - e.mu.Unlock() - } - return e.result -} - -// Get returns the cached result associated with key -// and reports whether there is such a result. -// -// If the result for key is being computed, Get does not wait for the computation to finish. -func (c *Cache[K, V]) Get(key K) (V, bool) { - entryIface, ok := c.m.Load(key) - if !ok { - return *new(V), false - } - e := entryIface.(*cacheEntry[V]) - if !e.done.Load() { - return *new(V), false - } - return e.result, true -} - -// Clear removes all entries in the cache. -// -// Concurrent calls to Get may return old values. Concurrent calls to Do -// may return old values or store results in entries that have been deleted. -// -// TODO(jayconrod): Delete this after the package cache clearing functions -// in internal/load have been removed. -func (c *Cache[K, V]) Clear() { - c.m.Range(func(key, value any) bool { - c.m.Delete(key) - return true - }) -} - -// Delete removes an entry from the map. It is safe to call Delete for an -// entry that does not exist. Delete will return quickly, even if the result -// for a key is still being computed; the computation will finish, but the -// result won't be accessible through the cache. -// -// TODO(jayconrod): Delete this after the package cache clearing functions -// in internal/load have been removed. -func (c *Cache[K, V]) Delete(key K) { - c.m.Delete(key) -} - -// DeleteIf calls pred for each key in the map. If pred returns true for a key, -// DeleteIf removes the corresponding entry. If the result for a key is -// still being computed, DeleteIf will remove the entry without waiting for -// the computation to finish. The result won't be accessible through the cache. -// -// TODO(jayconrod): Delete this after the package cache clearing functions -// in internal/load have been removed. -func (c *Cache[K, V]) DeleteIf(pred func(key K) bool) { - c.m.Range(func(key, _ any) bool { - if key := key.(K); pred(key) { - c.Delete(key) - } - return true - }) -} diff --git a/cue/load/internal/par/work_test.go b/cue/load/internal/par/work_test.go deleted file mode 100644 index 9d96ffae50c..00000000000 --- a/cue/load/internal/par/work_test.go +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright 2018 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package par - -import ( - "sync/atomic" - "testing" - "time" -) - -func TestWork(t *testing.T) { - var w Work[int] - - const N = 10000 - n := int32(0) - w.Add(N) - w.Do(100, func(i int) { - atomic.AddInt32(&n, 1) - if i >= 2 { - w.Add(i - 1) - w.Add(i - 2) - } - w.Add(i >> 1) - w.Add((i >> 1) ^ 1) - }) - if n != N+1 { - t.Fatalf("ran %d items, expected %d", n, N+1) - } -} - -func TestWorkParallel(t *testing.T) { - for tries := 0; tries < 10; tries++ { - var w Work[int] - const N = 100 - for i := 0; i < N; i++ { - w.Add(i) - } - start := time.Now() - var n int32 - w.Do(N, func(x int) { - time.Sleep(1 * time.Millisecond) - atomic.AddInt32(&n, +1) - }) - if n != N { - t.Fatalf("par.Work.Do did not do all the work") - } - if time.Since(start) < N/2*time.Millisecond { - return - } - } - t.Fatalf("par.Work.Do does not seem to be parallel") -} - -func TestCache(t *testing.T) { - var cache Cache[int, int] - - n := 1 - v := cache.Do(1, func() int { n++; return n }) - if v != 2 { - t.Fatalf("cache.Do(1) did not run f") - } - v = cache.Do(1, func() int { n++; return n }) - if v != 2 { - t.Fatalf("cache.Do(1) ran f again!") - } - v = cache.Do(2, func() int { n++; return n }) - if v != 3 { - t.Fatalf("cache.Do(2) did not run f") - } - v = cache.Do(1, func() int { n++; return n }) - if v != 2 { - t.Fatalf("cache.Do(1) did not returned saved value from original cache.Do(1)") - } -} diff --git a/cue/load/internal/registrytest/fileio.go b/cue/load/internal/registrytest/fileio.go new file mode 100644 index 00000000000..3fedc6d16cd --- /dev/null +++ b/cue/load/internal/registrytest/fileio.go @@ -0,0 +1,50 @@ +package registrytest + +import ( + "bytes" + "io" + "os" + "path" + "time" + + "golang.org/x/tools/txtar" +) + +// txtarFileIO implements mod/zip.FileIO[txtar.File]. +type txtarFileIO struct{} + +func (txtarFileIO) Path(f txtar.File) string { + return f.Name +} + +func (txtarFileIO) Lstat(f txtar.File) (os.FileInfo, error) { + return txtarFileInfo{f}, nil +} + +func (txtarFileIO) Open(f txtar.File) (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(f.Data)), nil +} + +func (txtarFileIO) Mode() os.FileMode { + return 0o444 +} + +type txtarFileInfo struct { + f txtar.File +} + +func (fi txtarFileInfo) Name() string { + return path.Base(fi.f.Name) +} + +func (fi txtarFileInfo) Size() int64 { + return int64(len(fi.f.Data)) +} + +func (fi txtarFileInfo) Mode() os.FileMode { + return 0o644 +} + +func (fi txtarFileInfo) ModTime() time.Time { return time.Time{} } +func (fi txtarFileInfo) IsDir() bool { return false } +func (fi txtarFileInfo) Sys() interface{} { return nil } diff --git a/cue/load/internal/registrytest/registry.go b/cue/load/internal/registrytest/registry.go index 9c2d869d08f..67cbc6f0cf9 100644 --- a/cue/load/internal/registrytest/registry.go +++ b/cue/load/internal/registrytest/registry.go @@ -2,42 +2,85 @@ package registrytest import ( "bytes" + "context" "fmt" "io" - "net/http" "net/http/httptest" - "os" - "path" "strings" - "time" - "golang.org/x/mod/module" - "golang.org/x/mod/semver" - "golang.org/x/mod/zip" + "cuelabs.dev/go/oci/ociregistry/ocimem" + "cuelabs.dev/go/oci/ociregistry/ociserver" "golang.org/x/tools/txtar" "cuelang.org/go/cue" "cuelang.org/go/cue/cuecontext" + "cuelang.org/go/internal/mod/modfile" + "cuelang.org/go/internal/mod/modregistry" + "cuelang.org/go/internal/mod/module" + "cuelang.org/go/internal/mod/zip" ) // New starts a registry instance that serves modules found inside the -// _registry path inside ar. The protocol that it serves is that of the -// Go proxy, documented here: https://go.dev/ref/mod#goproxy-protocol +// _registry path inside ar. It serves the OCI registry protocol. // // Each module should be inside a directory named path_vers, where // slashes in path have been replaced with underscores and should // contain a cue.mod/module.cue file holding the module info. // // The Registry should be closed after use. -func New(ar *txtar.Archive) *Registry { - h, err := newHandler(ar) +func New(ar *txtar.Archive) (*Registry, error) { + srv := httptest.NewServer(ociserver.New(ocimem.New(), nil)) + client, err := modregistry.NewClient(srv.URL, "cue/") if err != nil { - panic(err) + return nil, fmt.Errorf("cannot make client: %v", err) + } + mods, err := getModules(ar) + if err != nil { + return nil, fmt.Errorf("invalid modules: %v", err) + } + if err := pushContent(client, mods); err != nil { + return nil, fmt.Errorf("cannot push modules: %v", err) } - srv := httptest.NewServer(h) return &Registry{ srv: srv, + }, nil +} + +func pushContent(client *modregistry.Client, mods map[module.Version]*moduleContent) error { + pushed := make(map[module.Version]bool) + for v := range mods { + err := visitDepthFirst(mods, v, func(v module.Version, m *moduleContent) error { + if pushed[v] { + return nil + } + var zipContent bytes.Buffer + if err := m.writeZip(&zipContent); err != nil { + return err + } + if err := client.PutModule(context.Background(), v, bytes.NewReader(zipContent.Bytes()), int64(zipContent.Len())); err != nil { + return err + } + pushed[v] = true + return nil + }) + if err != nil { + return err + } } + return nil +} + +func visitDepthFirst(mods map[module.Version]*moduleContent, v module.Version, f func(module.Version, *moduleContent) error) error { + m := mods[v] + if m == nil { + return fmt.Errorf("no module found for version %v", v) + } + for _, depv := range m.modFile.DepVersions() { + if err := visitDepthFirst(mods, depv, f); err != nil { + return err + } + } + return f(v, m) } type Registry struct { @@ -57,7 +100,7 @@ type handler struct { modules []*moduleContent } -func newHandler(ar *txtar.Archive) (*handler, error) { +func getModules(ar *txtar.Archive) (map[module.Version]*moduleContent, error) { ctx := cuecontext.New() modules := make(map[string]*moduleContent) for _, f := range ar.Files { @@ -67,7 +110,7 @@ func newHandler(ar *txtar.Archive) (*handler, error) { } modver, rest, ok := strings.Cut(path, "/") if !ok { - return nil, fmt.Errorf("cannot have regular file inside _registry") + return nil, fmt.Errorf("_registry should only contain directories, but found regular file %q", path) } content := modules[modver] if content == nil { @@ -80,252 +123,59 @@ func newHandler(ar *txtar.Archive) (*handler, error) { }) } for modver, content := range modules { - if err := content.initVersion(ctx, modver); err != nil { - return nil, fmt.Errorf("cannot determine version for module in %q: %v", modver, err) + if err := content.init(ctx, modver); err != nil { + return nil, fmt.Errorf("cannot initialize module %q: %v", modver, err) } } - mods := make([]*moduleContent, 0, len(modules)) + byVer := map[module.Version]*moduleContent{} for _, m := range modules { - mods = append(mods, m) + byVer[m.version] = m } - return &handler{ - modules: mods, - }, nil -} - -var modulePath = cue.MakePath(cue.Str("module")) - -func modulePathFromModFile(ctx *cue.Context, data []byte) (string, error) { - v := ctx.CompileBytes(data) - if err := v.Err(); err != nil { - return "", fmt.Errorf("invalid module.cue syntax: %v", err) - } - v = v.LookupPath(modulePath) - s, err := v.String() - if err != nil { - return "", fmt.Errorf("cannot get module value from module.cue file: %v", err) - } - if s == "" { - return "", fmt.Errorf("empty module directive") - } - // TODO check for valid module path? - return s, nil -} - -func (r *handler) ServeHTTP(w http.ResponseWriter, req *http.Request) { - mreq, err := parseReq(req.URL.Path) - if err != nil { - http.Error(w, fmt.Sprintf("cannot parse request %q: %v", req.URL.Path, err), http.StatusBadRequest) - return - } - switch mreq.kind { - case reqMod: - data, err := r.getMod(mreq) - if err != nil { - http.Error(w, fmt.Sprintf("cannot get module: %v", err), http.StatusNotFound) - return - } - // TODO content type - w.Write(data) - case reqZip: - data, err := r.getZip(mreq) - if err != nil { - // TODO this can fail for non-NotFound reasons too. - http.Error(w, fmt.Sprintf("cannot get module contents: %v", err), http.StatusNotFound) - return - } - // TODO content type - w.Header().Set("Content-Type", "application/zip") - w.Write(data) - default: - http.Error(w, "not implemented yet", http.StatusInternalServerError) - } -} - -func (r *handler) getMod(req *request) ([]byte, error) { - for _, m := range r.modules { - if m.version == req.version { - return m.getMod(), nil - } - } - return nil, fmt.Errorf("no module found for %v", req.version) -} - -func (r *handler) getZip(req *request) ([]byte, error) { - for _, m := range r.modules { - if m.version == req.version { - // TODO write this to somewhere else temporary before - // writing to HTTP response. - var buf bytes.Buffer - if err := m.writeZip(&buf); err != nil { - return nil, err - } - return buf.Bytes(), nil - } - } - return nil, fmt.Errorf("no module found for %v", req.version) + return byVer, nil } type moduleContent struct { version module.Version files []txtar.File + modFile *modfile.File } func (c *moduleContent) writeZip(w io.Writer) error { - files := make([]zip.File, len(c.files)) - for i := range c.files { - files[i] = zipFile{&c.files[i]} - } - return zip.Create(w, c.version, files) + return zip.Create[txtar.File](w, c.version, c.files, txtarFileIO{}) } -type zipFile struct { - f *txtar.File -} - -// Path implements zip.File.Path. -func (f zipFile) Path() string { - return f.f.Name -} - -// Lstat implements zip.File.Lstat. -func (f zipFile) Lstat() (os.FileInfo, error) { - return f, nil -} - -func (f zipFile) Open() (io.ReadCloser, error) { - return io.NopCloser(bytes.NewReader(f.f.Data)), nil -} - -// Name implements fs.FileInfo.Name. -func (f zipFile) Name() string { - return path.Base(f.f.Name) -} - -// Mode implements fs.FileInfo.Mode. -func (f zipFile) Mode() os.FileMode { - return 0 -} - -// Size implements fs.FileInfo.Size. -func (f zipFile) Size() int64 { - return int64(len(f.f.Data)) -} - -func (f zipFile) IsDir() bool { - return false -} -func (f zipFile) ModTime() time.Time { - return time.Time{} -} -func (f zipFile) Sys() any { - return nil -} - -func (c *moduleContent) getMod() []byte { - for _, f := range c.files { - if f.Name == "cue.mod/module.cue" { - return f.Data - } - } - panic(fmt.Errorf("no module.cue file found in %v", c.version)) -} - -func (c *moduleContent) initVersion(ctx *cue.Context, versDir string) error { +func (c *moduleContent) init(ctx *cue.Context, versDir string) error { + found := false for _, f := range c.files { if f.Name != "cue.mod/module.cue" { continue } - mod, err := modulePathFromModFile(ctx, f.Data) + modf, err := modfile.Parse(f.Data, f.Name) if err != nil { - return fmt.Errorf("invalid module file in %q: %v", path.Join(versDir, f.Name), err) + return err } - if c.version.Path != "" { + if found { return fmt.Errorf("multiple module.cue files") } - c.version.Path = mod - mod = strings.ReplaceAll(mod, "/", "_") + "_" + modp, _, ok := module.SplitPathVersion(modf.Module) + if !ok { + return fmt.Errorf("module %q does not contain major version", modf.Module) + } + mod := strings.ReplaceAll(modp, "/", "_") + "_" vers := strings.TrimPrefix(versDir, mod) if len(vers) == len(versDir) { - return fmt.Errorf("module path %q in module.cue does not match directory %q", c.version.Path, versDir) + return fmt.Errorf("module path %q in module.cue does not match directory %q", modf.Module, versDir) } - if !semver.IsValid(vers) { - return fmt.Errorf("module version %q is not valid", vers) + v, err := module.NewVersion(modf.Module, vers) + if err != nil { + return fmt.Errorf("cannot make module version: %v", err) } - c.version.Version = vers + c.version = v + c.modFile = modf + found = true } - if c.version.Path == "" { + if !found { return fmt.Errorf("no module.cue file found in %q", versDir) } return nil } - -type reqKind int - -const ( - reqInvalid reqKind = iota - reqLatest - reqList - reqMod - reqZip - reqInfo -) - -type request struct { - version module.Version - kind reqKind -} - -func parseReq(urlPath string) (*request, error) { - urlPath = strings.TrimPrefix(urlPath, "/") - i := strings.LastIndex(urlPath, "/@") - if i == -1 { - return nil, fmt.Errorf("no @ found in path") - } - if i == 0 { - return nil, fmt.Errorf("empty module name in path") - } - var req request - mod, rest := urlPath[:i], urlPath[i+1:] - req.version.Path = mod - qual, rest, ok := strings.Cut(rest, "/") - if qual == "@latest" { - if ok { - return nil, fmt.Errorf("invalid @latest request") - } - // $base/$module/@latest - req.kind = reqLatest - return &req, nil - } - if qual != "@v" { - return nil, fmt.Errorf("invalid @ in request") - } - if !ok { - return nil, fmt.Errorf("no qualifier after @") - } - if rest == "list" { - // $base/$module/@v/list - req.kind = reqList - return &req, nil - } - i = strings.LastIndex(rest, ".") - if i == -1 { - return nil, fmt.Errorf("no . found after @") - } - vers, rest := rest[:i], rest[i+1:] - if len(vers) == 0 { - return nil, fmt.Errorf("empty version string") - } - req.version.Version = vers - switch rest { - case "info": - req.kind = reqInfo - case "mod": - req.kind = reqMod - case "zip": - req.kind = reqZip - default: - return nil, fmt.Errorf("unknown request kind %q", rest) - } - return &req, nil -} diff --git a/cue/load/internal/registrytest/registry_test.go b/cue/load/internal/registrytest/registry_test.go index cd18555f126..176bdd554be 100644 --- a/cue/load/internal/registrytest/registry_test.go +++ b/cue/load/internal/registrytest/registry_test.go @@ -1,17 +1,18 @@ package registrytest import ( + "context" "fmt" - "io" - "net/http" "os" "path" "path/filepath" - "strconv" "strings" "testing" "golang.org/x/tools/txtar" + + "cuelang.org/go/internal/mod/modregistry" + "cuelang.org/go/internal/mod/module" ) func TestRegistry(t *testing.T) { @@ -31,7 +32,10 @@ func TestRegistry(t *testing.T) { t.Fatal(err) } t.Run(strings.TrimSuffix(name, ".txtar"), func(t *testing.T) { - r := New(ar) + r, err := New(ar) + if err != nil { + t.Fatal(err) + } defer r.Close() runTest(t, r.URL(), string(ar.Comment), ar) }) @@ -39,8 +43,11 @@ func TestRegistry(t *testing.T) { } func runTest(t *testing.T, registry string, script string, ar *txtar.Archive) { - var resp *http.Response - var respBody []byte + ctx := context.Background() + client, err := modregistry.NewClient(registry, "cue/") + if err != nil { + t.Fatal(err) + } for _, line := range strings.Split(script, "\n") { if line == "" || line[0] == '#' { continue @@ -50,37 +57,28 @@ func runTest(t *testing.T, registry string, script string, ar *txtar.Archive) { t.Fatalf("invalid line %q", line) } switch args[0] { - case "GET": - if len(args) != 2 { - t.Fatalf("usage: GET $url") + case "modfile": + if len(args) != 3 { + t.Fatalf("usage: getmod $version $wantFile") } - resp1, err := http.Get(registry + "/" + args[1]) + mv, err := module.ParseVersion(args[1]) if err != nil { - t.Fatalf("GET failed: %v", err) + t.Fatalf("invalid version %q in getmod", args[1]) } - respBody, _ = io.ReadAll(resp1.Body) - resp1.Body.Close() - resp = resp1 - case "body": - if len(args) != 3 { - t.Fatalf("usage: body code file") + m, err := client.GetModule(ctx, mv) + if err != nil { + t.Fatal(err) } - wantCode, err := strconv.Atoi(args[1]) + gotData, err := m.ModuleFile(ctx) if err != nil { - t.Fatalf("invalid code %q", args[1]) + t.Fatal(err) } - wantBody, err := getFile(ar, args[2]) + wantData, err := getFile(ar, args[2]) if err != nil { t.Fatalf("cannot open file for body comparison: %v", err) } - if resp == nil { - t.Fatalf("no previous GET request to check body against") - } - if resp.StatusCode != wantCode { - t.Errorf("unexpected GET response code; got %v want %v", wantCode, resp.StatusCode) - } - if string(respBody) != string(wantBody) { - t.Errorf("unexpected GET response\ngot %q\nwant %q", respBody, wantBody) + if string(gotData) != string(wantData) { + t.Errorf("unexpected GET response\ngot %q\nwant %q", gotData, wantData) } default: t.Fatalf("unknown command %q", line) diff --git a/cue/load/internal/registrytest/testdata/simple.txtar b/cue/load/internal/registrytest/testdata/simple.txtar index 17882c135d6..5925b64cf7f 100644 --- a/cue/load/internal/registrytest/testdata/simple.txtar +++ b/cue/load/internal/registrytest/testdata/simple.txtar @@ -1,15 +1,12 @@ -GET example.com/@v/v0.0.1.mod -body 200 _registry/example.com_v0.0.1/cue.mod/module.cue - -GET foo.com/bar/hello/@v/v0.2.3.mod -body 200 _registry/foo.com_bar_hello_v0.2.3/cue.mod/module.cue +modfile example.com@v0.0.1 _registry/example.com_v0.0.1/cue.mod/module.cue +modfile foo.com/bar/hello@v0.2.3 _registry/foo.com_bar_hello_v0.2.3/cue.mod/module.cue -- _registry/example.com_v0.0.1/cue.mod/module.cue -- -module: "example.com" +module: "example.com@v0" deps: { - "foo.com/bar/hello": v: "v0.2.3" - "bar.com": v: "v0.5.0" + "foo.com/bar/hello@v0": v: "v0.2.3" + "bar.com@v0": v: "v0.5.0" } -- _registry/example.com_v0.0.1/top.cue -- package main @@ -18,11 +15,11 @@ import a "foo.com/bar/hello" a main: "main" -- _registry/foo.com_bar_hello_v0.2.3/cue.mod/module.cue -- -module: "foo.com/bar/hello" +module: "foo.com/bar/hello@v0" deps: { - "bar.com": v: "v0.0.2" - "baz.org": v: "v0.10.1" + "bar.com@v0": v: "v0.0.2" + "baz.org@v0": v: "v0.10.1" } -- _registry/foo.com_bar_hello_v0.2.3/x.cue -- package hello @@ -34,9 +31,9 @@ import ( a b -- _registry/bar.com_v0.0.2/cue.mod/module.cue -- -module: "bar.com" +module: "bar.com@v0" -deps: "baz.org": v: "v0.1.2" +deps: "baz.org@v0": v: "v0.1.2" -- _registry/bar.com_v0.0.2/bar/x.cue -- package bar @@ -45,10 +42,14 @@ import a "baz.org/baz" "bar.com": "v0.0.2" a -- _registry/baz.org_v0.10.1/cue.mod/module.cue -- -module: "baz.org" +module: "baz.org@v0" -- _registry/baz.org_v0.10.1/baz.cue -- -"baz.org": "v0.10.1" +"baz.org@v0": "v0.10.1" -- _registry/baz.org_v0.1.2/cue.mod/module.cue -- -module: "baz.org" -"baz.org": "v0.1.2" +module: "baz.org@v0" + +-- _registry/bar.com_v0.5.0/cue.mod/module.cue -- +module: "bar.com@v0" +-- _registry/bar.com_v0.5.0/bar.cue -- +"bar.com@v0": "v0.5.0" diff --git a/cue/load/internal/slices/slices.go b/cue/load/internal/slices/slices.go deleted file mode 100644 index a0adcf4926b..00000000000 --- a/cue/load/internal/slices/slices.go +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2023 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// TODO: Replace with slices package when it lands in standard library. - -package slices - -// Clip removes unused capacity from the slice, returning s[:len(s):len(s)]. -func Clip[S ~[]E, E any](s S) S { - return s[:len(s):len(s)] -} diff --git a/cue/load/loader_test.go b/cue/load/loader_test.go index a96cc00469e..5f72d71eed3 100644 --- a/cue/load/loader_test.go +++ b/cue/load/loader_test.go @@ -45,7 +45,6 @@ func TestLoad(t *testing.T) { badModCfg := &Config{ Dir: testMod("badmod"), } - type loadTest struct { cfg *Config args []string @@ -56,13 +55,7 @@ func TestLoad(t *testing.T) { testCases := []loadTest{{ cfg: badModCfg, args: args("."), - want: `err: module: invalid module.cue file: 2 errors in empty disjunction: -module: invalid module.cue file: conflicting values 123 and "" (mismatched types int and string): - $cueroot/cue/load/moduleschema.cue:4:20 - $CWD/testdata/badmod/cue.mod/module.cue:2:9 -module: invalid module.cue file: conflicting values 123 and =~"^[^@]+$" (mismatched types int and string): - $cueroot/cue/load/moduleschema.cue:4:10 - $cueroot/cue/load/moduleschema.cue:21:21 + want: `err: module: cannot use value 123 (type int) as string: $CWD/testdata/badmod/cue.mod/module.cue:2:9 path: "" module: "" @@ -83,8 +76,7 @@ display:. files: $CWD/testdata/testmod/test.cue imports: - mod.test/test/sub: $CWD/testdata/testmod/sub/sub.cue`, - }, { + mod.test/test/sub: $CWD/testdata/testmod/sub/sub.cue`}, { // Even though the directory is called testdata, the last path in // the module is test. So "package test" is correctly the default // package of this directory. @@ -98,8 +90,7 @@ display:. files: $CWD/testdata/testmod/test.cue imports: - mod.test/test/sub: $CWD/testdata/testmod/sub/sub.cue`, - }, { + mod.test/test/sub: $CWD/testdata/testmod/sub/sub.cue`}, { // TODO: // - path incorrect, should be mod.test/test/other:main. cfg: dirCfg, @@ -110,8 +101,7 @@ path: "" module: mod.test/test root: $CWD/testdata/testmod dir: "" -display:""`, - }, { +display:""`}, { cfg: dirCfg, args: args("./anon"), want: `err: build constraints exclude all CUE files in ./anon: @@ -120,8 +110,7 @@ path: mod.test/test/anon module: mod.test/test root: $CWD/testdata/testmod dir: $CWD/testdata/testmod/anon -display:./anon`, - }, { +display:./anon`}, { // TODO: // - paths are incorrect, should be mod.test/test/other:main. cfg: dirCfg, @@ -134,8 +123,7 @@ root: $CWD/testdata/testmod dir: $CWD/testdata/testmod/other display:./other files: - $CWD/testdata/testmod/other/main.cue`, - }, { + $CWD/testdata/testmod/other/main.cue`}, { // TODO: // - incorrect path, should be mod.test/test/hello:test cfg: dirCfg, @@ -149,8 +137,7 @@ files: $CWD/testdata/testmod/test.cue $CWD/testdata/testmod/hello/test.cue imports: - mod.test/test/sub: $CWD/testdata/testmod/sub/sub.cue`, - }, { + mod.test/test/sub: $CWD/testdata/testmod/sub/sub.cue`}, { // TODO: // - incorrect path, should be mod.test/test/hello:test cfg: dirCfg, @@ -164,8 +151,7 @@ files: $CWD/testdata/testmod/test.cue $CWD/testdata/testmod/hello/test.cue imports: - mod.test/test/sub: $CWD/testdata/testmod/sub/sub.cue`, - }, { + mod.test/test/sub: $CWD/testdata/testmod/sub/sub.cue`}, { // TODO: // - incorrect path, should be mod.test/test/hello:test cfg: dirCfg, @@ -178,8 +164,7 @@ path: mod.test/test/hello:nonexist module: mod.test/test root: $CWD/testdata/testmod dir: $CWD/testdata/testmod/hello -display:mod.test/test/hello:nonexist`, - }, { +display:mod.test/test/hello:nonexist`}, { cfg: dirCfg, args: args("./anon.cue", "./other/anon.cue"), want: `path: "" @@ -189,8 +174,7 @@ dir: $CWD/testdata/testmod display:command-line-arguments files: $CWD/testdata/testmod/anon.cue - $CWD/testdata/testmod/other/anon.cue`, - }, { + $CWD/testdata/testmod/other/anon.cue`}, { cfg: dirCfg, // Absolute file is normalized. args: args(filepath.Join(testMod("testmod"), "anon.cue")), @@ -200,8 +184,7 @@ root: $CWD/testdata/testmod dir: $CWD/testdata/testmod display:command-line-arguments files: - $CWD/testdata/testmod/anon.cue`, - }, { + $CWD/testdata/testmod/anon.cue`}, { cfg: dirCfg, args: args("-"), want: `path: "" @@ -210,12 +193,11 @@ root: $CWD/testdata/testmod dir: $CWD/testdata/testmod display:command-line-arguments files: - -`, - }, { + -`}, { // NOTE: dir should probably be set to $CWD/testdata, but either way. cfg: dirCfg, args: args("non-existing"), - want: `err: cannot find package "non-existing" + want: `err: implied package identifier "non-existing" from import path "non-existing" is not valid path: non-existing module: mod.test/test root: $CWD/testdata/testmod @@ -242,8 +224,7 @@ files: $CWD/testdata/testmod/imports/imports.cue imports: mod.test/catch: $CWD/testdata/testmod/cue.mod/pkg/mod.test/catch/catch.cue - mod.test/helper:helper1: $CWD/testdata/testmod/cue.mod/pkg/mod.test/helper/helper1.cue`, - }, { + mod.test/helper:helper1: $CWD/testdata/testmod/cue.mod/pkg/mod.test/helper/helper1.cue`}, { cfg: dirCfg, args: args("./toolonly"), want: `path: mod.test/test/toolonly:foo @@ -252,8 +233,7 @@ root: $CWD/testdata/testmod dir: $CWD/testdata/testmod/toolonly display:./toolonly files: - $CWD/testdata/testmod/toolonly/foo_tool.cue`, - }, { + $CWD/testdata/testmod/toolonly/foo_tool.cue`}, { cfg: &Config{ Dir: testdataDir, }, @@ -266,8 +246,7 @@ path: mod.test/test/toolonly:foo module: mod.test/test root: $CWD/testdata/testmod dir: $CWD/testdata/testmod/toolonly -display:./toolonly`, - }, { +display:./toolonly`}, { cfg: &Config{ Dir: testdataDir, Tags: []string{"prod"}, @@ -279,8 +258,7 @@ root: $CWD/testdata/testmod dir: $CWD/testdata/testmod/tags display:./tags files: - $CWD/testdata/testmod/tags/prod.cue`, - }, { + $CWD/testdata/testmod/tags/prod.cue`}, { cfg: &Config{ Dir: testdataDir, Tags: []string{"prod", "foo=bar"}, @@ -292,8 +270,7 @@ root: $CWD/testdata/testmod dir: $CWD/testdata/testmod/tags display:./tags files: - $CWD/testdata/testmod/tags/prod.cue`, - }, { + $CWD/testdata/testmod/tags/prod.cue`}, { cfg: &Config{ Dir: testdataDir, Tags: []string{"prod"}, @@ -308,8 +285,7 @@ path: mod.test/test/tagsbad module: mod.test/test root: $CWD/testdata/testmod dir: $CWD/testdata/testmod/tagsbad -display:./tagsbad`, - }, { +display:./tagsbad`}, { cfg: &Config{ Dir: testdataDir, }, @@ -324,8 +300,7 @@ root: $CWD/testdata/testmod dir: $CWD/testdata/testmod/cycle display:./cycle files: - $CWD/testdata/testmod/cycle/cycle.cue`, - }} + $CWD/testdata/testmod/cycle/cycle.cue`}} tdtest.Run(t, testCases, func(t *tdtest.T, tc *loadTest) { pkgs := Instances(tc.args, tc.cfg) diff --git a/cue/load/module.go b/cue/load/module.go index e6d40b1b91f..7247de57df7 100644 --- a/cue/load/module.go +++ b/cue/load/module.go @@ -1,50 +1,20 @@ package load import ( - _ "embed" + "context" + "fmt" "io" "path/filepath" "strings" - "golang.org/x/mod/module" - "golang.org/x/mod/semver" - "cuelang.org/go/cue/errors" - "cuelang.org/go/cue/load/internal/mvs" "cuelang.org/go/cue/token" + "cuelang.org/go/internal/mod/modfile" + "cuelang.org/go/internal/mod/module" + "cuelang.org/go/internal/mod/mvs" + "golang.org/x/mod/semver" ) -//go:embed moduleschema.cue -var moduleSchema []byte - -type modFile struct { - Module string `json:"module"` - Deps map[string]*modDep -} - -// versions returns all the modules that are dependended on by -// the module file. -func (mf *modFile) versions() []module.Version { - if len(mf.Deps) == 0 { - // It's important to return nil here because otherwise the - // "mistake: chose versions" panic in mvs will trigger - // on an empty version list. - return nil - } - vs := make([]module.Version, 0, len(mf.Deps)) - for m, dep := range mf.Deps { - vs = append(vs, module.Version{ - Path: m, - Version: dep.Version, - }) - } - return vs -} - -type modDep struct { - Version string `json:"v"` -} - // loadModule loads the module file, resolves and downloads module // dependencies. It sets c.Module if it's empty or checks it for // consistency with the module file otherwise. @@ -69,7 +39,11 @@ func (c *Config) loadModule() error { if err != nil { return err } - mf, err := parseModuleFile(data, mod) + parseModFile := modfile.ParseNonStrict + if c.Registry == "" { + parseModFile = modfile.ParseLegacy + } + mf, err := parseModFile(data, mod) if err != nil { return err } @@ -87,26 +61,53 @@ func (c *Config) loadModule() error { } type dependencies struct { - versions []module.Version + mainModule *modfile.File + versions []module.Version } // lookup returns the module corresponding to the given import path, and the relative path // of the package beneath that. // // It assumes that modules are not nested. -func (deps *dependencies) lookup(pkgPath importPath) (v module.Version, subPath string, ok bool) { +func (deps *dependencies) lookup(pkgPath importPath) (v module.Version, subPath string, err error) { + type answer struct { + v module.Version + subPath string + } + var possible []answer for _, dep := range deps.versions { - if subPath, ok := isParent(importPath(dep.Path), pkgPath); ok { - return dep, subPath, true + if subPath, ok := isParent(dep, pkgPath); ok { + possible = append(possible, answer{dep, subPath}) } } - return module.Version{}, "", false + switch len(possible) { + case 0: + return module.Version{}, "", fmt.Errorf("no dependency found for import path %q", pkgPath) + case 1: + return possible[0].v, possible[0].subPath, nil + } + var found *answer + for i, a := range possible { + dep, ok := deps.mainModule.Deps[a.v.Path()] + if ok && dep.Default { + if found != nil { + // More than one default. + // TODO this should be impossible and checked by modfile. + return module.Version{}, "", fmt.Errorf("more than one default module for import path %q", pkgPath) + } + found = &possible[i] + } + } + if found == nil { + return module.Version{}, "", fmt.Errorf("no default module found for import path %q", pkgPath) + } + return found.v, found.subPath, nil } // resolveDependencies resolves all the versions of all the modules in the given module file, // using regClient to fetch dependency information. -func resolveDependencies(mainModFile *modFile, regClient *registryClient) (*dependencies, error) { - vs, err := mvs.BuildList(mainModFile.versions(), &mvsReqs{ +func resolveDependencies(mainModFile *modfile.File, regClient *registryClient) (*dependencies, error) { + vs, err := mvs.BuildList[module.Version](mainModFile.DepVersions(), &mvsReqs{ mainModule: mainModFile, regClient: regClient, }) @@ -114,27 +115,29 @@ func resolveDependencies(mainModFile *modFile, regClient *registryClient) (*depe return nil, err } return &dependencies{ - versions: vs, + mainModule: mainModFile, + versions: vs, }, nil } // mvsReqs implements mvs.Reqs by fetching information using // regClient. type mvsReqs struct { - mainModule *modFile + module.Versions + mainModule *modfile.File regClient *registryClient } // Required implements mvs.Reqs.Required. func (reqs *mvsReqs) Required(m module.Version) (vs []module.Version, err error) { - if m.Path == reqs.mainModule.Module { - return reqs.mainModule.versions(), nil + if m.Path() == reqs.mainModule.Module { + return reqs.mainModule.DepVersions(), nil } - mf, err := reqs.regClient.fetchModFile(m) + mf, err := reqs.regClient.fetchModFile(context.TODO(), m) if err != nil { return nil, err } - return mf.versions(), nil + return mf.DepVersions(), nil } // Required implements mvs.Reqs.Max. @@ -164,17 +167,28 @@ func cmpVersion(v1, v2 string) int { return semver.Compare(v1, v2) } -// isParent reports whether the module modPath contains the package with the given +// isParent reports whether the module modv contains the package with the given // path, and if so, returns its relative path within that module. -func isParent(modPath, pkgPath importPath) (subPath string, ok bool) { - if !strings.HasPrefix(string(pkgPath), string(modPath)) { - return "", false +func isParent(modv module.Version, pkgPath importPath) (subPath string, ok bool) { + modBase := modv.BasePath() + pkgBase, pkgMajor, pkgHasVersion := module.SplitPathVersion(string(pkgPath)) + if !pkgHasVersion { + pkgBase = string(pkgPath) } - if len(pkgPath) == len(modPath) { - return ".", true + + if !strings.HasPrefix(pkgBase, modBase) { + return "", false } - if pkgPath[len(modPath)] != '/' { + if len(pkgBase) == len(modBase) { + subPath = "." + } else if pkgBase[len(modBase)] != '/' { return "", false + } else { + subPath = pkgBase[len(modBase)+1:] + } + // It's potentially a match, but we need to check the major version too. + if !pkgHasVersion || semver.Major(modv.Version()) == pkgMajor { + return subPath, true } - return string(pkgPath[len(modPath)+1:]), true + return "", false } diff --git a/cue/load/module_test.go b/cue/load/module_test.go index 4630ad14d38..de971ba8541 100644 --- a/cue/load/module_test.go +++ b/cue/load/module_test.go @@ -16,7 +16,10 @@ func TestModuleFetch(t *testing.T) { Name: "modfetch", } test.Run(t, func(t *cuetxtar.Test) { - r := registrytest.New(t.Archive) + r, err := registrytest.New(t.Archive) + if err != nil { + t.Fatal(err) + } defer r.Close() t.LoadConfig.Registry = r.URL() ctx := cuecontext.New() diff --git a/cue/load/registry.go b/cue/load/registry.go index 933e5f3d64c..14976dbf529 100644 --- a/cue/load/registry.go +++ b/cue/load/registry.go @@ -1,29 +1,25 @@ package load import ( + "context" "fmt" "io" - "net/http" "os" "path" "path/filepath" - "golang.org/x/mod/module" - "golang.org/x/mod/zip" - - "cuelang.org/go/cue" - "cuelang.org/go/cue/errors" - "cuelang.org/go/cue/parser" - "cuelang.org/go/cue/token" - "cuelang.org/go/internal/core/runtime" + "cuelang.org/go/internal/mod/modfile" + "cuelang.org/go/internal/mod/modregistry" + "cuelang.org/go/internal/mod/module" + "cuelang.org/go/internal/mod/zip" ) // registryClient implements the protocol for talking to // the registry server. type registryClient struct { // TODO caching - registryURL string - cacheDir string + client *modregistry.Client + cacheDir string } // newRegistryClient returns a registry client that talks to @@ -31,21 +27,25 @@ type registryClient struct { // in the given cache directory. It assumes that information // in the registry is immutable, so if it's in the cache, a module // will not be downloaded again. -func newRegistryClient(registryURL, cacheDir string) *registryClient { - return ®istryClient{ - registryURL: registryURL, - cacheDir: cacheDir, +func newRegistryClient(registryHost string, cacheDir string) (*registryClient, error) { + client, err := modregistry.NewClient(registryHost, "cue/") // TODO configurable prefix + if err != nil { + return nil, err } + return ®istryClient{ + client: client, + cacheDir: cacheDir, + }, nil } // fetchModFile returns the parsed contents of the cue.mod/module.cue file // for the given module. -func (c *registryClient) fetchModFile(m module.Version) (*modFile, error) { - data, err := c.fetchRawModFile(m) +func (c *registryClient) fetchModFile(ctx context.Context, m module.Version) (*modfile.File, error) { + data, err := c.fetchRawModFile(ctx, m) if err != nil { return nil, err } - mf, err := parseModuleFile(data, path.Join(m.Path, "cue.mod/module.cue")) + mf, err := modfile.Parse(data, path.Join(m.Path(), "cue.mod/module.cue")) if err != nil { return nil, err } @@ -54,40 +54,31 @@ func (c *registryClient) fetchModFile(m module.Version) (*modFile, error) { // fetchModFile returns the contents of the cue.mod/module.cue file // for the given module without parsing it. -func (c *registryClient) fetchRawModFile(m module.Version) ([]byte, error) { - resp, err := http.Get(c.registryURL + "/" + m.Path + "/@v/" + m.Version + ".mod") +func (c *registryClient) fetchRawModFile(ctx context.Context, mv module.Version) ([]byte, error) { + m, err := c.client.GetModule(ctx, mv) if err != nil { return nil, err } - defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("cannot get HTTP response body: %v", err) - } - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("module.cue HTTP GET request failed: %s", body) - } - return body, nil + return m.ModuleFile(ctx) } // getModContents downloads the module with the given version // and returns the directory where it's stored. -func (c *registryClient) getModContents(m module.Version) (string, error) { - modPath := filepath.Join(c.cacheDir, fmt.Sprintf("%s@%s", m.Path, m.Version)) +func (c *registryClient) getModContents(ctx context.Context, mv module.Version) (string, error) { + modPath := filepath.Join(c.cacheDir, mv.String()) if _, err := os.Stat(modPath); err == nil { return modPath, nil } - // TODO synchronize parallel invocations - resp, err := http.Get(c.registryURL + "/" + m.Path + "/@v/" + m.Version + ".zip") + m, err := c.client.GetModule(ctx, mv) if err != nil { return "", err } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return "", fmt.Errorf("module.cue HTTP GET request failed: %s", body) + r, err := m.GetZip(ctx) + if err != nil { + return "", err } - zipfile := filepath.Join(c.cacheDir, m.String()+".zip") + defer r.Close() + zipfile := filepath.Join(c.cacheDir, mv.String()+".zip") if err := os.MkdirAll(filepath.Dir(zipfile), 0o777); err != nil { return "", fmt.Errorf("cannot create parent directory for zip file: %v", err) } @@ -97,40 +88,11 @@ func (c *registryClient) getModContents(m module.Version) (string, error) { } defer f.Close() // TODO check error on close - if _, err := io.Copy(f, resp.Body); err != nil { + if _, err := io.Copy(f, r); err != nil { return "", fmt.Errorf("cannot copy data to zip file %q: %v", zipfile, err) } - if err := zip.Unzip(modPath, m, zipfile); err != nil { - return "", fmt.Errorf("cannot unzip %v: %v", m, err) + if err := zip.Unzip(modPath, mv, zipfile); err != nil { + return "", fmt.Errorf("cannot unzip %v: %v", mv, err) } return modPath, nil } - -// parseModuleFile parses a cue.mod/module.cue file. -// TODO move this to be closer to the modFile type definition. -func parseModuleFile(data []byte, filename string) (*modFile, error) { - file, err := parser.ParseFile(filename, data) - if err != nil { - return nil, errors.Wrapf(err, token.NoPos, "invalid module.cue file %q", data) - } - // TODO disallow non-data-mode CUE. - - ctx := (*cue.Context)(runtime.New()) - schemav := ctx.CompileBytes(moduleSchema, cue.Filename("$cueroot/cue/load/moduleschema.cue")) - if err := schemav.Validate(); err != nil { - return nil, errors.Wrapf(err, token.NoPos, "internal error: invalid CUE module.cue schema") - } - v := ctx.BuildFile(file) - if err := v.Validate(cue.Concrete(true)); err != nil { - return nil, errors.Wrapf(err, token.NoPos, "invalid module.cue file") - } - v = v.Unify(schemav) - if err := v.Validate(); err != nil { - return nil, errors.Wrapf(err, token.NoPos, "invalid module.cue file") - } - var mf modFile - if err := v.Decode(&mf); err != nil { - return nil, errors.Wrapf(err, token.NoPos, "internal error: cannot decode into modFile struct (\nfile %q\ncontents %q\nvalue %#v\n)", filename, data, v) - } - return &mf, nil -} diff --git a/cue/load/testdata/testfetch/depnotfound.txtar b/cue/load/testdata/testfetch/depnotfound.txtar index 152aea05255..1aba878e12a 100644 --- a/cue/load/testdata/testfetch/depnotfound.txtar +++ b/cue/load/testdata/testfetch/depnotfound.txtar @@ -1,10 +1,9 @@ -- out/modfetch/error -- -instance: example.com@v0.0.1: module.cue HTTP GET request failed: cannot get module: no module found for example.com@v0.0.1 - +instance: example.com@v0.0.1: module example.com@v0.0.1: error response: 404 Not Found: repository name not known to registry -- cue.mod/module.cue -- -module: "main.org" +module: "main.org@v0" -deps: "example.com": v: "v0.0.1" +deps: "example.com@v0": v: "v0.0.1" -- main.cue -- package main diff --git a/cue/load/testdata/testfetch/simple.txtar b/cue/load/testdata/testfetch/simple.txtar index 33a3ba41424..5358463ab14 100644 --- a/cue/load/testdata/testfetch/simple.txtar +++ b/cue/load/testdata/testfetch/simple.txtar @@ -1,105 +1,106 @@ -- out/modfetch -- { - main: "main" - "foo.com/bar/hello": "v0.2.3" - "bar.com": "v0.5.0" - "baz.org": "v0.10.1" - "example.com": "v0.0.1" + main: "main" + "foo.com/bar/hello@v0": "v0.2.3" + "bar.com@v0": "v0.5.0" + "baz.org@v0": "v0.10.1" + "example.com@v0": "v0.0.1" } -- cue.mod/module.cue -- module: "main.org" -deps: "example.com": v: "v0.0.1" +deps: "example.com@v0": v: "v0.0.1" -- main.cue -- package main -import "example.com:main" +import "example.com@v0:main" main -- _registry/example.com_v0.0.1/cue.mod/module.cue -- -module: "example.com" +module: "example.com@v0" deps: { - "foo.com/bar/hello": v: "v0.2.3" - "bar.com": v: "v0.5.0" + "foo.com/bar/hello@v0": v: "v0.2.3" + "bar.com@v0": v: "v0.5.0" } -- _registry/example.com_v0.0.1/top.cue -- package main +// Note: import without a major version takes +// the major version from the module.cue file. import a "foo.com/bar/hello" a main: "main" -"example.com": "v0.0.1" - +"example.com@v0": "v0.0.1" -- _registry/foo.com_bar_hello_v0.2.3/cue.mod/module.cue -- -module: "foo.com/bar/hello" +module: "foo.com/bar/hello@v0" deps: { - "bar.com": v: "v0.0.2" - "baz.org": v: "v0.10.1" + "bar.com@v0": v: "v0.0.2" + "baz.org@v0": v: "v0.10.1" } -- _registry/foo.com_bar_hello_v0.2.3/x.cue -- package hello import ( - a "bar.com/bar" - b "baz.org:baz" + a "bar.com/bar@v0" + b "baz.org@v0:baz" ) -"foo.com/bar/hello": "v0.2.3" +"foo.com/bar/hello@v0": "v0.2.3" a b -- _registry/bar.com_v0.0.2/cue.mod/module.cue -- -module: "bar.com" -deps: "baz.org": v: "v0.0.2" +module: "bar.com@v0" +deps: "baz.org@v0": v: "v0.0.2" -- _registry/bar.com_v0.0.2/bar/x.cue -- package bar -import a "baz.org:baz" -"bar.com": "v0.0.2" +import a "baz.org@v0:baz" +"bar.com@v0": "v0.0.2" a -- _registry/bar.com_v0.5.0/cue.mod/module.cue -- -module: "bar.com" -deps: "baz.org": v: "v0.5.0" +module: "bar.com@v0" +deps: "baz.org@v0": v: "v0.5.0" -- _registry/bar.com_v0.5.0/bar/x.cue -- package bar -import a "baz.org:baz" -"bar.com": "v0.5.0" +import a "baz.org@v0:baz" +"bar.com@v0": "v0.5.0" a -- _registry/baz.org_v0.0.2/cue.mod/module.cue -- -module: "baz.org" +module: "baz.org@v0" -- _registry/baz.org_v0.0.2/baz.cue -- package baz -"baz.org": "v0.0.2" +"baz.org@v0": "v0.0.2" -- _registry/baz.org_v0.1.2/cue.mod/module.cue -- -module: "baz.org" +module: "baz.org@v0" -- _registry/baz.org_v0.1.2/baz.cue -- package baz -"baz.org": "v0.1.2" +"baz.org@v0": "v0.1.2" -- _registry/baz.org_v0.5.0/cue.mod/module.cue -- -module: "baz.org" +module: "baz.org@v0" -- _registry/baz.org_v0.5.0/baz.cue -- package baz -"baz.org": "v0.5.0" +"baz.org@v0": "v0.5.0" -- _registry/baz.org_v0.10.1/cue.mod/module.cue -- -module: "baz.org" +module: "baz.org@v0" -- _registry/baz.org_v0.10.1/baz.cue -- package baz -"baz.org": "v0.10.1" +"baz.org@v0": "v0.10.1" diff --git a/internal/mod/modfile/modfile_test.go b/internal/mod/modfile/modfile_test.go index ddf9df78d76..0202e18fca1 100644 --- a/internal/mod/modfile/modfile_test.go +++ b/internal/mod/modfile/modfile_test.go @@ -27,12 +27,14 @@ import ( var tests = []struct { testName string + parse func(modfile []byte, filename string) (*File, error) data string wantError string want *File wantVersions []module.Version }{{ testName: "NoDeps", + parse: Parse, data: ` module: "foo.com/bar@v0" `, @@ -41,6 +43,7 @@ module: "foo.com/bar@v0" }, }, { testName: "WithDeps", + parse: Parse, data: ` language: version: "v0.4.3" module: "foo.com/bar@v0" @@ -64,6 +67,7 @@ deps: "other.com/something@v0": v: "v0.2.3" wantVersions: parseVersions("example.com@v1.2.3", "other.com/something@v0.2.3"), }, { testName: "MisspelledLanguageVersionField", + parse: Parse, data: ` langugage: version: "v0.4.3" module: "foo.com/bar@v0" @@ -74,12 +78,14 @@ module: "foo.com/bar@v0" module.cue:2:1`, }, { testName: "InvalidLanguageVersion", + parse: Parse, data: ` language: version: "vblah" module: "foo.com/bar@v0"`, wantError: `language version "vblah" in module.cue is not well formed`, }, { testName: "InvalidDepVersion", + parse: Parse, data: ` module: "foo.com/bar@v1" deps: "example.com@v1": v: "1.2.3" @@ -87,6 +93,7 @@ deps: "example.com@v1": v: "1.2.3" wantError: `invalid module.cue file module.cue: cannot make version from module "example.com@v1", version "1.2.3": version "1.2.3" \(of module "example.com@v1"\) is not well formed`, }, { testName: "NonCanonicalVersion", + parse: Parse, data: ` module: "foo.com/bar@v1" deps: "example.com@v1": v: "v1.2" @@ -94,12 +101,14 @@ deps: "example.com@v1": v: "v1.2" wantError: `invalid module.cue file module.cue: cannot make version from module "example.com@v1", version "v1.2": version "v1.2" \(of module "example.com@v1"\) is not canonical`, }, { testName: "NonCanonicalModule", + parse: Parse, data: ` module: "foo.com/bar" `, wantError: `module path "foo.com/bar" in module.cue does not contain major version`, }, { testName: "NonCanonicalDep", + parse: Parse, data: ` module: "foo.com/bar@v1" deps: "example.com": v: "v1.2.3" @@ -107,17 +116,45 @@ deps: "example.com": v: "v1.2.3" wantError: `invalid module.cue file module.cue: no major version in "example.com"`, }, { testName: "MismatchedMajorVersion", + parse: Parse, data: ` module: "foo.com/bar@v1" deps: "example.com@v1": v: "v0.1.2" `, wantError: `invalid module.cue file module.cue: cannot make version from module "example.com@v1", version "v0.1.2": mismatched major version suffix in "example.com@v1" \(version v0.1.2\)`, +}, { + testName: "NonStrictNoMajorVersions", + parse: ParseNonStrict, + data: ` +module: "foo.com/bar" +deps: "example.com": v: "v1.2.3" +`, + want: &File{ + Module: "foo.com/bar", + Deps: map[string]*Dep{ + "example.com": { + Version: "v1.2.3", + }, + }, + }, + wantVersions: parseVersions("example.com@v1.2.3"), +}, { + testName: "LegacyWithExtraFields", + parse: ParseLegacy, + data: ` +module: "foo.com/bar" +something: 4 +cue: lang: "xxx" +`, + want: &File{ + Module: "foo.com/bar", + }, }} func TestParse(t *testing.T) { for _, test := range tests { t.Run(test.testName, func(t *testing.T) { - f, err := Parse([]byte(test.data), "module.cue") + f, err := test.parse([]byte(test.data), "module.cue") if test.wantError != "" { gotErr := strings.TrimSuffix(errors.Details(err, nil), "\n") qt.Assert(t, qt.Matches(gotErr, test.wantError)) diff --git a/internal/mod/modregistry/client_test.go b/internal/mod/modregistry/client_test.go index c807730ecf1..6facb867ea2 100644 --- a/internal/mod/modregistry/client_test.go +++ b/internal/mod/modregistry/client_test.go @@ -79,7 +79,13 @@ deps: "example.com@v1": v: "v1.2.3" deps: "other.com/something@v0": v: "v0.2.3" -- x.cue -- -x: 42 +package bar + +import ( + a "example.com" + "other.com/something" +) +x: a.foo + something.bar ` ctx := context.Background() mv := module.MustParseVersion("foo.com/bar@v0.5.100") diff --git a/internal/mod/zip/zip.go b/internal/mod/zip/zip.go index 9b527fbe1a3..bf411a26d10 100644 --- a/internal/mod/zip/zip.go +++ b/internal/mod/zip/zip.go @@ -8,7 +8,7 @@ // to ensure that module zip files can be extracted consistently on supported // platforms and file systems. // -// • All file paths within a zip file must be valid (see cuelang.org/go/mod/module.CheckFilePath). +// • All file paths within a zip file must be valid (see cuelang.org/go/internal/mod/module.CheckFilePath). // // • No two file paths may be equal under Unicode case-folding (see // strings.EqualFold).