Add xcryptobackendswap, implement x/crypto SHA3 using backend

.gitmodules

@@ -1,3 +1,8 @@
 [submodule "go"]
 	path = go
 	url = https://github.com/golang/go
+	urlInternal = https://dnceng@dev.azure.com/dnceng/internal/_git/microsoft-go-mirror
+[submodule "eng/modules/golang.org/x/crypto"]
+	path = modules/golang.org/x/crypto
+	url = https://github.com/dagood/golang_xcrypto
+	urlInternal = https://dnceng@dev.azure.com/dnceng/internal/_git/microsoft-go-xcrypto-fork-placeholder

eng/_core/cmd/build/build.go

@@ -50,6 +50,15 @@ func main() {
 		&o.Refresh, "refresh", false,
 		"Refresh Go submodule: clean untracked files, reset tracked files, and apply patches before building.\n"+
 			"For more refresh options, use the top level 'submodule-refresh' command instead of 'build'.")
+	flag.BoolVar(
+		&o.SkipMSMod, "skipmsmod", false,
+		"Skip creating the _ms_mod directory with the x/crypto fork to be used by the XCryptoSwap experiment.\n"+
+			"Note: tests will fail. The patched standard library test suite will notice if the the backend isn't used for x/crypto.\n"+
+			"A alternate way to fix tests without _ms_mod generation is to set GOMSMODROOT to a manually created fork.")
+	flag.BoolVar(
+		&o.CleanMSMod, "cleanmsmod", false,
+		"Remove _ms_mod directory when the build is complete.\n"+
+			"This may be useful during development to run tests without polluting the Go submodule's working tree.")
 
 	flag.StringVar(&o.Experiment, "experiment", "", "Include this string in GOEXPERIMENT.")
 
@@ -84,6 +93,8 @@ type options struct {
 	PackBuild  bool
 	PackSource bool
 	Refresh    bool
+	SkipMSMod  bool
+	CleanMSMod bool
 	Experiment string
 
 	MaxMakeAttempts int
@@ -120,6 +131,21 @@ func build(o *options) error {
 		}
 	}
 
+	if !o.SkipMSMod {
+		fmt.Println("---- Generating _ms_mod...")
+		if err := submodule.GenerateMSMod(rootDir); err != nil {
+			return fmt.Errorf("failed to generate _ms_mod: %v", err)
+		}
+	}
+	if o.CleanMSMod {
+		defer func() {
+			fmt.Println("---- Cleanup: removing _ms_mod directory...")
+			if err := submodule.RemoveMSMod(rootDir); err != nil {
+				fmt.Printf("---- Error removing _ms_mod directory: %v\n", err)
+			}
+		}()
+	}
+
 	// Get the target platform information. If the environment variable is different from the
 	// runtime value, this means we're doing a cross-compiled build. These values are used for
 	// capability checks and to make sure that if Pack is enabled, the output archive is formatted

eng/_core/cmd/submodule-refresh/submodule-refresh.go

@@ -20,7 +20,7 @@ applies patches to the stage by default, or optionally as commits.
 
 var commits = flag.Bool("commits", false, "Apply the patches as commits.")
 var skipPatch = flag.Bool("skip-patch", false, "Skip applying patches.")
-var origin = flag.String("origin", "", "Use this origin instead of the default defined in '.gitmodules' to fetch the repository.")
+var internal = flag.Bool("internal", false, "Use the .gitmodules urlInternal instead of url to clone submodules.")
 var shallow = flag.Bool("shallow", false, "Clone the submodule with depth 1.")
 var fetchBearerToken = flag.String("fetch-bearer-token", "", "Use this bearer token to fetch the submodule repository.")
 
@@ -50,7 +50,7 @@ func main() {
 }
 
 func refresh(rootDir string) error {
-	if err := submodule.Init(rootDir, *origin, *fetchBearerToken, *shallow); err != nil {
+	if err := submodule.Init(rootDir, *internal, *fetchBearerToken, *shallow); err != nil {
 		return err
 	}
 

eng/_core/go.mod

@@ -4,4 +4,8 @@
 
 module github.com/microsoft/go/_core
 
-go 1.16
+go 1.18
+
+require github.com/microsoft/go-infra v0.0.0-20230921065609-974951356d4d
+
+require golang.org/x/tools v0.1.12 // indirect

eng/_core/go.sum

@@ -0,0 +1,5 @@
+github.com/microsoft/go-infra v0.0.0-20230921065609-974951356d4d h1:2dYhK+YQIwmN5n/AImhvU6Krn48vgWGCcuvwLb1wEJ4=
+github.com/microsoft/go-infra v0.0.0-20230921065609-974951356d4d/go.mod h1:hGTlq0ZBZVmZwHgV+JezVdi8jZ2f4/UydeYvMVjAJLM=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s=
+golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=

eng/_core/submodule/submodule.go

@@ -9,17 +9,25 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strings"
+
+	"github.com/microsoft/go-infra/executil"
+	"github.com/microsoft/go-infra/xcryptofork"
 )
 
 // Init initializes and updates the submodule, but does not clean it. This func offers more options
 // for initialization than Reset. If origin is defined, fetch the submodule from there instead of
 // the default defined in '.gitmodules'. If fetchBearerToken is nonempty, use it as a bearer token
 // during the fetch. If shallow is true, clone the submodule with depth 1.
-func Init(rootDir, origin, fetchBearerToken string, shallow bool) error {
+func Init(rootDir string, internal bool, fetchBearerToken string, shallow bool) error {
 	// Update the submodule commit, and initialize if it hasn't been done already.
 	command := []string{"git"}
-	if origin != "" {
-		command = append(command, "-c", "submodule.go.url="+origin)
+	if internal {
+		var err error
+		command, err = appendURLInternalGitConfigArgs(rootDir, command)
+		if err != nil {
+			return err
+		}
 	}
 	if fetchBearerToken != "" {
 		command = append(command, "-c", "http.extraheader=AUTHORIZATION: bearer "+fetchBearerToken)
@@ -44,23 +52,10 @@ func Reset(rootDir string) error {
 		return err
 	}
 
-	// Find toplevel directories (Git working tree roots) for the outer repo and what we expect to
-	// be the Go submodule. If the toplevel directory is the same for both, make sure not to clean!
-	// The submodule likely wasn't set up properly, and cleaning could result in unexpectedly losing
-	// work in the outer repo when the command spills over.
-	rootToplevel, err := getToplevel(rootDir)
-	if err != nil {
-		return err
-	}
-	goToplevel, err := getToplevel(goDir)
-	if err != nil {
+	if err := assertSubmoduleInitialized(rootDir, goDir); err != nil {
 		return err
 	}
 
-	if rootToplevel == goToplevel {
-		return fmt.Errorf("go submodule (%v) toplevel is the same as root (%v) toplevel: %v", goDir, rootDir, goToplevel)
-	}
-
 	// Reset the index and working directory. This doesn't clean up new untracked files.
 	if err := run(goDir, "git", "reset", "--hard"); err != nil {
 		return err
@@ -74,6 +69,170 @@ func Reset(rootDir string) error {
 	return nil
 }
 
+// GenerateMSMod creates the _ms_mod directory with x/crypto fork and generates
+// the crypto backend proxies based on the backends currently in the Go
+// submodule source tree.
+func GenerateMSMod(rootDir string) error {
+	cryptoSrcDir := filepath.Join(rootDir, "modules", "golang.org", "x", "crypto")
+	if err := assertSubmoduleInitialized(rootDir, cryptoSrcDir); err != nil {
+		return err
+	}
+	goDir := filepath.Join(rootDir, "go")
+	if err := assertSubmoduleInitialized(rootDir, goDir); err != nil {
+		return err
+	}
+	// Target directory for the x/crypto fork.
+	cryptoDir := filepath.Join(goDir, "_ms_mod", "golang.org", "x", "crypto")
+	// Clean it up. No prompt: there shouldn't be any need to do dev work in
+	// this directory.
+	if err := os.RemoveAll(cryptoDir); err != nil {
+		return err
+	}
+	if err := os.MkdirAll(cryptoDir, 0o777); err != nil {
+		return err
+	}
+	if err := xcryptofork.GitCheckoutTo(cryptoSrcDir, cryptoDir); err != nil {
+		return err
+	}
+	// Generate the backend proxies and the nobackend file based on the backends
+	// in the active Go tree. The placeholder in x/crypto is ignored: it's only
+	// there so the x/crypto fork will compile outside this context.
+	backendDir := filepath.Join(goDir, "src", "crypto", "internal", "backend")
+	backends, err := xcryptofork.FindBackendFiles(backendDir)
+	if err != nil {
+		return fmt.Errorf("failed to find backend files in %q: %v", backendDir, err)
+	}
+	proxyDir := filepath.Join(cryptoDir, "internal", "backend")
+	if err := os.RemoveAll(proxyDir); err != nil {
+		return err
+	}
+	// First, find the nobackend. It defines the API for the backend proxies.
+	const nobackendBase = "nobackend.go"
+	var backendAPI *xcryptofork.BackendFile
+	for _, b := range backends {
+		if filepath.Base(b.Filename) == nobackendBase {
+			if err := b.APITrim(); err != nil {
+				return fmt.Errorf("failed to trim %q into an API for proxies: %v", b.Filename, err)
+			}
+			// If someone uses the x/crypto fork but doesn't use a backend, they
+			// will need this nobackend.go for their build to succeed.
+			if err := writeBackend(b, filepath.Join(proxyDir, nobackendBase)); err != nil {
+				return fmt.Errorf("failed to write API based on %q: %v", b.Filename, err)
+			}
+			backendAPI = b
+			break
+		}
+	}
+	if backendAPI == nil {
+		return fmt.Errorf("%q not found in %v", nobackendBase, backendDir)
+	}
+	// Create a proxy for each backend.
+	for _, b := range backends {
+		if b == backendAPI {
+			continue
+		}
+		proxy, err := b.ProxyAPI(backendAPI)
+		if err != nil {
+			return fmt.Errorf("failed to turn %q into a proxy: %v", b.Filename, err)
+		}
+		if err := writeBackend(proxy, filepath.Join(proxyDir, filepath.Base(b.Filename))); err != nil {
+			return fmt.Errorf("failed to write proxy based on %q: %v", b.Filename, err)
+		}
+	}
+	// Also generate a placeholder based on nobackend for the x/crypto fork
+	// itself. This is intended to be used during dev so x/crypto will build,
+	// but avoid making it specific to the proxies generated for our fork.
+	backendAPI.Constraint = ""
+	placeholderDir := filepath.Join(rootDir, "eng", "artifacts", "xcrypto-backend-placeholder")
+	if err := os.MkdirAll(placeholderDir, 0o777); err != nil {
+		return err
+	}
+	if err := writeBackend(backendAPI, filepath.Join(placeholderDir, "placeholder.go")); err != nil {
+		return fmt.Errorf("failed to write placeholder backend based on %q: %v", backendAPI.Filename, err)
+	}
+	return nil
+}
+
+func RemoveMSMod(rootDir string) error {
+	msMod := filepath.Join(rootDir, "go", "_ms_mod")
+	entries, err := os.ReadDir(msMod)
+	if err != nil {
+		return err
+	}
+	for _, e := range entries {
+		if !e.IsDir() {
+			continue
+		}
+		if err := os.RemoveAll(filepath.Join(msMod, e.Name())); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func writeBackend(b xcryptofork.FormattedWriterTo, path string) error {
+	if err := os.MkdirAll(filepath.Dir(path), 0o777); err != nil {
+		return err
+	}
+	apiFile, err := os.Create(path)
+	if err != nil {
+		return err
+	}
+	err = b.Format(apiFile)
+	if err2 := apiFile.Close(); err == nil {
+		err = err2
+	}
+	return err
+}
+
+func appendURLInternalGitConfigArgs(rootDir string, args []string) ([]string, error) {
+	cmd := exec.Command(
+		"git", "config",
+		"-f", ".gitmodules",
+		"-z", // Null char separator: avoid confusion with newlines in values.
+		"--get-regexp", `submodule\..*\.urlInternal`)
+	cmd.Dir = rootDir
+	out, err := executil.CombinedOutput(cmd)
+	if err != nil {
+		return nil, err
+	}
+	pairs := strings.Split(out, "\x00")
+	for _, pair := range pairs {
+		key, value, ok := strings.Cut(pair, "\n")
+		if !ok {
+			return nil, fmt.Errorf("invalid key-value pair: %v", pair)
+		}
+		args = append(args, "-c", strings.TrimSuffix(key, "Internal")+"="+value)
+	}
+	return args, nil
+}
+
+// assertSubmoduleInitialized runs a basic check to ensure the submodule within
+// the specified root repo is initialized. It finds toplevel directories (Git
+// working tree roots) for the outer repo and what we expect to be the Go
+// submodule. If the toplevel directory is the same for both, the submodule
+// likely wasn't set up properly, and (e.g.) cleaning the submodule dir could
+// result in unexpectedly losing work in the rootDir when the command spills
+// over and affects the outer repo.
+//
+// There may be other ways to check whether the submodule is initialized, but
+// this check at the very least helps at the most painful potential symptom of
+// an uninitialized submodule: lost work.
+func assertSubmoduleInitialized(rootDir, submoduleRootDir string) error {
+	rootToplevel, err := getToplevel(rootDir)
+	if err != nil {
+		return err
+	}
+	submoduleToplevel, err := getToplevel(submoduleRootDir)
+	if err != nil {
+		return err
+	}
+	if rootToplevel == submoduleToplevel {
+		return fmt.Errorf("go submodule (%v) toplevel is the same as root (%v) toplevel: %v", submoduleToplevel, rootDir, submoduleToplevel)
+	}
+	return nil
+}
+
 func getToplevel(dir string) (string, error) {
 	c := exec.Command("git", "rev-parse", "--show-toplevel")
 	c.Dir = dir