Modify imports package to avoid disk IO, update to use Go 1.5 and js packages.

Generate zstdlib.go, include Go 1.5, skip syscall.

Use a modified version of cmd/api tool to include gopherjs/js package.

Add go generate step for imports package to update.sh.

Author: dmitshur

playground/internal/imports/fix.go

@@ -5,16 +5,10 @@
 package imports
 
 import (
-	"fmt"
 	"go/ast"
-	"go/build"
-	"go/parser"
 	"go/token"
-	"os"
 	"path"
-	"path/filepath"
 	"strings"
-	"sync"
 
 	"golang.org/x/tools/go/ast/astutil"
 )
@@ -145,243 +139,41 @@ func fixImports(fset *token.FileSet, f *ast.File) (added []string, err error) {
 }
 
 // importPathToName returns the package name for the given import path.
-var importPathToName = importPathToNameGoPath
+var importPathToName = importPathToNameBasic
 
 // importPathToNameBasic assumes the package name is the base of import path.
 func importPathToNameBasic(importPath string) (packageName string) {
 	return path.Base(importPath)
 }
 
-// importPathToNameGoPath finds out the actual package name, as declared in its .go files.
-// If there's a problem, it falls back to using importPathToNameBasic.
-func importPathToNameGoPath(importPath string) (packageName string) {
-	if buildPkg, err := build.Import(importPath, "", 0); err == nil {
-		return buildPkg.Name
-	} else {
-		return importPathToNameBasic(importPath)
-	}
-}
-
 type pkg struct {
 	importpath string // full pkg import path, e.g. "net/http"
 	dir        string // absolute file path to pkg directory e.g. "/usr/lib/go/src/fmt"
 }
 
-var pkgIndexOnce sync.Once
-
-var pkgIndex struct {
-	sync.Mutex
-	m map[string][]pkg // shortname => []pkg, e.g "http" => "net/http"
-}
-
-// gate is a semaphore for limiting concurrency.
-type gate chan struct{}
-
-func (g gate) enter() { g <- struct{}{} }
-func (g gate) leave() { <-g }
-
-// fsgate protects the OS & filesystem from too much concurrency.
-// Too much disk I/O -> too many threads -> swapping and bad scheduling.
-var fsgate = make(gate, 8)
-
-func loadPkgIndex() {
-	pkgIndex.Lock()
-	pkgIndex.m = make(map[string][]pkg)
-	pkgIndex.Unlock()
-
-	var wg sync.WaitGroup
-	for _, path := range build.Default.SrcDirs() {
-		fsgate.enter()
-		f, err := os.Open(path)
-		if err != nil {
-			fsgate.leave()
-			fmt.Fprint(os.Stderr, err)
-			continue
-		}
-		children, err := f.Readdir(-1)
-		f.Close()
-		fsgate.leave()
-		if err != nil {
-			fmt.Fprint(os.Stderr, err)
-			continue
-		}
-		for _, child := range children {
-			if child.IsDir() {
-				wg.Add(1)
-				go func(path, name string) {
-					defer wg.Done()
-					loadPkg(&wg, path, name)
-				}(path, child.Name())
-			}
-		}
-	}
-	wg.Wait()
-}
-
-func loadPkg(wg *sync.WaitGroup, root, pkgrelpath string) {
-	importpath := filepath.ToSlash(pkgrelpath)
-	dir := filepath.Join(root, importpath)
-
-	fsgate.enter()
-	defer fsgate.leave()
-	pkgDir, err := os.Open(dir)
-	if err != nil {
-		return
-	}
-	children, err := pkgDir.Readdir(-1)
-	pkgDir.Close()
-	if err != nil {
-		return
-	}
-	// hasGo tracks whether a directory actually appears to be a
-	// Go source code directory. If $GOPATH == $HOME, and
-	// $HOME/src has lots of other large non-Go projects in it,
-	// then the calls to importPathToName below can be expensive.
-	hasGo := false
-	for _, child := range children {
-		// Avoid .foo, _foo, and testdata directory trees.
-		name := child.Name()
-		if name == "" || name[0] == '.' || name[0] == '_' || name == "testdata" {
-			continue
-		}
-		if strings.HasSuffix(name, ".go") {
-			hasGo = true
-		}
-		if child.IsDir() {
-			wg.Add(1)
-			go func(root, name string) {
-				defer wg.Done()
-				loadPkg(wg, root, name)
-			}(root, filepath.Join(importpath, name))
-		}
-	}
-	if hasGo {
-		shortName := importPathToName(importpath)
-		pkgIndex.Lock()
-		pkgIndex.m[shortName] = append(pkgIndex.m[shortName], pkg{
-			importpath: importpath,
-			dir:        dir,
-		})
-		pkgIndex.Unlock()
-	}
-
-}
-
-// loadExports returns a list exports for a package.
-var loadExports = loadExportsGoPath
-
-func loadExportsGoPath(dir string) map[string]bool {
-	exports := make(map[string]bool)
-	buildPkg, err := build.ImportDir(dir, 0)
-	if err != nil {
-		if strings.Contains(err.Error(), "no buildable Go source files in") {
-			return nil
-		}
-		fmt.Fprintf(os.Stderr, "could not import %q: %v\n", dir, err)
-		return nil
-	}
-	fset := token.NewFileSet()
-	for _, files := range [...][]string{buildPkg.GoFiles, buildPkg.CgoFiles} {
-		for _, file := range files {
-			f, err := parser.ParseFile(fset, filepath.Join(dir, file), nil, 0)
-			if err != nil {
-				fmt.Fprintf(os.Stderr, "could not parse %q: %v\n", file, err)
-				continue
-			}
-			for name := range f.Scope.Objects {
-				if ast.IsExported(name) {
-					exports[name] = true
-				}
-			}
-		}
-	}
-	return exports
-}
-
 // findImport searches for a package with the given symbols.
 // If no package is found, findImport returns "".
 // Declared as a variable rather than a function so goimports can be easily
 // extended by adding a file with an init function.
-var findImport = findImportGoPath
-
-func findImportGoPath(pkgName string, symbols map[string]bool) (string, bool, error) {
-	// Fast path for the standard library.
-	// In the common case we hopefully never have to scan the GOPATH, which can
-	// be slow with moving disks.
-	if pkg, rename, ok := findImportStdlib(pkgName, symbols); ok {
-		return pkg, rename, nil
-	}
-
-	// TODO(sameer): look at the import lines for other Go files in the
-	// local directory, since the user is likely to import the same packages
-	// in the current Go file.  Return rename=true when the other Go files
-	// use a renamed package that's also used in the current file.
-
-	pkgIndexOnce.Do(loadPkgIndex)
-
-	// Collect exports for packages with matching names.
-	var wg sync.WaitGroup
-	var pkgsMu sync.Mutex // guards pkgs
-	// full importpath => exported symbol => True
-	// e.g. "net/http" => "Client" => True
-	pkgs := make(map[string]map[string]bool)
-	pkgIndex.Lock()
-	for _, pkg := range pkgIndex.m[pkgName] {
-		wg.Add(1)
-		go func(importpath, dir string) {
-			defer wg.Done()
-			exports := loadExports(dir)
-			if exports != nil {
-				pkgsMu.Lock()
-				pkgs[importpath] = exports
-				pkgsMu.Unlock()
-			}
-		}(pkg.importpath, pkg.dir)
-	}
-	pkgIndex.Unlock()
-	wg.Wait()
-
-	// Filter out packages missing required exported symbols.
-	for symbol := range symbols {
-		for importpath, exports := range pkgs {
-			if !exports[symbol] {
-				delete(pkgs, importpath)
-			}
-		}
-	}
-	if len(pkgs) == 0 {
-		return "", false, nil
-	}
-
-	// If there are multiple candidate packages, the shortest one wins.
-	// This is a heuristic to prefer the standard library (e.g. "bytes")
-	// over e.g. "github.com/foo/bar/bytes".
-	shortest := ""
-	for importPath := range pkgs {
-		if shortest == "" || len(importPath) < len(shortest) {
-			shortest = importPath
-		}
-	}
-	return shortest, false, nil
-}
+var findImport = findImportStdlib
 
 type visitFn func(node ast.Node) ast.Visitor
 
 func (fn visitFn) Visit(node ast.Node) ast.Visitor {
 	return fn(node)
 }
 
-func findImportStdlib(shortPkg string, symbols map[string]bool) (importPath string, rename, ok bool) {
+func findImportStdlib(shortPkg string, symbols map[string]bool) (importPath string, rename bool, err error) {
 	for symbol := range symbols {
 		path := stdlib[shortPkg+"."+symbol]
 		if path == "" {
-			return "", false, false
+			return "", false, nil
 		}
 		if importPath != "" && importPath != path {
 			// Ambiguous. Symbols pointed to different things.
-			return "", false, false
+			return "", false, nil
 		}
 		importPath = path
 	}
-	return importPath, false, importPath != ""
+	return importPath, false, nil
 }

playground/internal/imports/gen.go

@@ -0,0 +1,5 @@
+//go:generate go run mkapi.go -output=gopherjs.txt -gopath github.com/gopherjs/gopherjs/js
+//go:generate go run mkstdlib.go -output=zstdlib
+//go:generate rm gopherjs.txt
+
+package imports

playground/internal/imports/imports.go

@@ -4,7 +4,7 @@
 
 // Package imports implements a Go pretty-printer (like package "go/format")
 // that also adds or removes import statements as necessary.
-package imports // import "golang.org/x/tools/imports"
+package imports
 
 import (
 	"bufio"

playground/internal/imports/mkapi.go

@@ -1,8 +1,15 @@
+// +build generate
+
 // Copyright 2011 The Go Authors.  All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
 // Binary api computes the exported API of a set of Go packages.
+//
+// It's a copy of cmd/api tool, but with a patch to enable
+// operating on packages in GOPATH workspaces merged in
+// (https://github.com/golang/go/issues/4993#issuecomment-66075958),
+// and an output flag added for go generate usage.
 package main
 
 import (
@@ -29,12 +36,14 @@ import (
 
 // Flags
 var (
-	checkFile  = flag.String("c", "", "optional comma-separated filename(s) to check API against")
-	allowNew   = flag.Bool("allow_new", true, "allow API additions")
-	exceptFile = flag.String("except", "", "optional filename of packages that are allowed to change without triggering a failure in the tool")
-	nextFile   = flag.String("next", "", "optional filename of tentative upcoming API features for the next release. This file can be lazily maintained. It only affects the delta warnings from the -c file printed on success.")
-	verbose    = flag.Bool("v", false, "verbose debugging")
-	forceCtx   = flag.String("contexts", "", "optional comma-separated list of <goos>-<goarch>[-cgo] to override default contexts.")
+	output        = flag.String("output", "", "output file name; if empty, then print to stdout")
+	includeGopath = flag.Bool("gopath", false, "include GOPATH packages")
+	checkFile     = flag.String("c", "", "optional comma-separated filename(s) to check API against")
+	allowNew      = flag.Bool("allow_new", true, "allow API additions")
+	exceptFile    = flag.String("except", "", "optional filename of packages that are allowed to change without triggering a failure in the tool")
+	nextFile      = flag.String("next", "", "optional filename of tentative upcoming API features for the next release. This file can be lazily maintained. It only affects the delta warnings from the -c file printed on success.")
+	verbose       = flag.Bool("v", false, "verbose debugging")
+	forceCtx      = flag.String("contexts", "", "optional comma-separated list of <goos>-<goarch>[-cgo] to override default contexts.")
 )
 
 // contexts are the default contexts which are scanned, unless
@@ -126,6 +135,12 @@ func main() {
 	var pkgNames []string
 	if flag.NArg() > 0 {
 		pkgNames = flag.Args()
+	} else if *includeGopath {
+		alls, err := exec.Command("go", "list", "all").Output()
+		if err != nil {
+			log.Fatal(err)
+		}
+		pkgNames = strings.Fields(string(alls))
 	} else {
 		stds, err := exec.Command("go", "list", "std").Output()
 		if err != nil {
@@ -186,7 +201,19 @@ func main() {
 		}
 	}()
 
-	bw := bufio.NewWriter(os.Stdout)
+	var out io.Writer
+	switch *output {
+	case "":
+		out = os.Stdout
+	default:
+		f, err := os.OpenFile(*output, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+		if err != nil {
+			log.Fatalf("opening file %q for output: %v", *output, err)
+		}
+		defer f.Close()
+		out = f
+	}
+	bw := bufio.NewWriter(out)
 	defer bw.Flush()
 
 	if *checkFile == "" {
@@ -428,17 +455,27 @@ func (w *Walker) Import(name string) (*types.Package, error) {
 	}
 	w.imported[name] = &importing
 
-	// Determine package files.
-	dir := filepath.Join(w.root, filepath.FromSlash(name))
-	if fi, err := os.Stat(dir); err != nil || !fi.IsDir() {
-		log.Fatalf("no source in tree for package %q", pkg)
-	}
-
 	context := w.context
 	if context == nil {
 		context = &build.Default
 	}
 
+	// Determine package files.
+	var dir string
+	// if 'name' is not a std package, then include GOPATH
+	if pkg == nil && *includeGopath {
+		c_pkg, err := build.Default.Import(name, "", build.FindOnly)
+		if err != nil {
+			log.Fatalf("failed to find import %q", name)
+		}
+		dir = c_pkg.Dir
+	} else {
+		dir = filepath.Join(w.root, filepath.FromSlash(name))
+		if fi, err := os.Stat(dir); err != nil || !fi.IsDir() {
+			log.Fatalf("no source in tree for package %q (import name %q)", pkg, name)
+		}
+	}
+
 	// Look in cache.
 	// If we've already done an import with the same set
 	// of relevant tags, reuse the result.