Add .netrc support for remote http(s) paths

api/go.mod

@@ -3,10 +3,12 @@ module sigs.k8s.io/kustomize/api
 go 1.19
 
 require (
+	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d
 	github.com/evanphx/json-patch v4.11.0+incompatible
 	github.com/go-errors/errors v1.4.2
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/imdario/mergo v0.3.6
+	github.com/mitchellh/go-homedir v1.1.0
 	github.com/stretchr/testify v1.8.1
 	gopkg.in/yaml.v2 v2.4.0
 	k8s.io/kube-openapi v0.0.0-20230109183929-3758b55a6596

api/go.sum

@@ -1,5 +1,7 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas=
+github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
@@ -61,6 +63,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0=
 github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=

api/internal/localizer/localizer.go

@@ -366,7 +366,7 @@ func (lc *localizer) localizeFile(path string) (string, error) {
 // localizeFileWithContent writes content to the localized file path and returns the localized path.
 func (lc *localizer) localizeFileWithContent(path string, content []byte) (string, error) {
 	var locPath string
-	if loader.IsRemoteFile(path) {
+	if remote, _ := loader.IsRemoteFile(path); remote {
 		if lc.fSys.Exists(lc.root.Join(LocalizeDir)) {
 			return "", errors.Errorf("%s already contains %s needed to store file %q", lc.root, LocalizeDir, path)
 		}

api/internal/localizer/locloader.go

@@ -92,7 +92,7 @@ func (ll *Loader) Load(path string) ([]byte, error) {
 	if filepath.IsAbs(path) {
 		return nil, errors.Errorf("absolute paths not yet supported in alpha: file path %q is absolute", path)
 	}
-	if !loader.IsRemoteFile(path) && ll.local {
+	if remote, _ := loader.IsRemoteFile(path); !remote && ll.local {
 		cleanPath := cleanFilePath(ll.fSys, filesys.ConfirmedDir(ll.Root()), path)
 		cleanAbs := filepath.Join(ll.Root(), cleanPath)
 		dir := filesys.ConfirmedDir(filepath.Dir(cleanAbs))

api/loader/fileloader.go

@@ -9,9 +9,13 @@ import (
 	"log"
 	"net/http"
 	"net/url"
+	"os"
 	"path/filepath"
+	"runtime"
 	"strings"
 
+	"github.com/bgentry/go-netrc/netrc"
+	"github.com/mitchellh/go-homedir"
 	"sigs.k8s.io/kustomize/api/ifc"
 	"sigs.k8s.io/kustomize/api/internal/git"
 	"sigs.k8s.io/kustomize/kyaml/errors"
@@ -20,9 +24,9 @@ import (
 
 // IsRemoteFile returns whether path has a url scheme that kustomize allows for
 // remote files. See https://github.com/kubernetes-sigs/kustomize/blob/master/examples/remoteBuild.md
-func IsRemoteFile(path string) bool {
+func IsRemoteFile(path string) (bool, *url.URL) {
 	u, err := url.Parse(path)
-	return err == nil && (u.Scheme == "http" || u.Scheme == "https")
+	return err == nil && (u.Scheme == "http" || u.Scheme == "https"), u
 }
 
 // fileLoader is a kustomization's interface to files.
@@ -306,8 +310,8 @@ func (fl *fileLoader) errIfRepoCycle(newRepoSpec *git.RepoSpec) error {
 // else an error. Relative paths are taken relative
 // to the root.
 func (fl *fileLoader) Load(path string) ([]byte, error) {
-	if IsRemoteFile(path) {
-		return fl.httpClientGetContent(path)
+	if remote, u := IsRemoteFile(path); remote {
+		return fl.httpClientGetContent(u)
 	}
 	if !filepath.IsAbs(path) {
 		path = fl.root.Join(path)
@@ -319,21 +323,25 @@ func (fl *fileLoader) Load(path string) ([]byte, error) {
 	return fl.fSys.ReadFile(path)
 }
 
-func (fl *fileLoader) httpClientGetContent(path string) ([]byte, error) {
+func (fl *fileLoader) httpClientGetContent(u *url.URL) ([]byte, error) {
 	var hc *http.Client
 	if fl.http != nil {
 		hc = fl.http
 	} else {
 		hc = &http.Client{}
 	}
-	resp, err := hc.Get(path)
+	err := addAuthFromNetrc(u)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := hc.Get(u.String())
 	if err != nil {
 		return nil, errors.Wrap(err)
 	}
 	defer resp.Body.Close()
 	// response unsuccessful
 	if resp.StatusCode < 200 || resp.StatusCode > 299 {
-		_, err = git.NewRepoSpecFromURL(path)
+		_, err = git.NewRepoSpecFromURL(u.String())
 		if err == nil {
 			return nil, errors.Errorf("URL is a git repository")
 		}
@@ -347,3 +355,59 @@ func (fl *fileLoader) httpClientGetContent(path string) ([]byte, error) {
 func (fl *fileLoader) Cleanup() error {
 	return fl.cleaner()
 }
+
+// addAuthFromNetrc adds auth information to the URL from the user's
+// netrc file if it can be found. This will only add the auth info
+// if the URL doesn't already have auth info specified and the
+// the username is blank.
+func addAuthFromNetrc(u *url.URL) error {
+	// If the URL already has auth information, do nothing
+	if u.User != nil && u.User.Username() != "" {
+		return nil
+	}
+
+	// Get the netrc file path
+	path := os.Getenv("NETRC")
+	if path == "" {
+		filename := ".netrc"
+		if runtime.GOOS == "windows" {
+			filename = "_netrc"
+		}
+
+		var err error
+		path, err = homedir.Expand("~/" + filename)
+		if err != nil {
+			return err
+		}
+	}
+
+	// If the file is not a file, then do nothing
+	if fi, err := os.Stat(path); err != nil {
+		// File doesn't exist, do nothing
+		if os.IsNotExist(err) {
+			return nil
+		}
+
+		// Some other error!
+		return err
+	} else if fi.IsDir() {
+		// File is directory, ignore
+		return nil
+	}
+
+	// Load up the netrc file
+	netrc, err := netrc.ParseFile(path)
+	if err != nil {
+		return fmt.Errorf("Error parsing netrc file at %q: %s", path, err)
+	}
+
+	machine := netrc.FindMachine(u.Host)
+	if machine == nil {
+		// Machine not found, no problem
+		return nil
+	}
+
+	// Set the user info
+	u.User = url.UserPassword(machine.Login, machine.Password)
+	return nil
+}

api/loader/fileloader_test.go

@@ -51,7 +51,8 @@ func TestIsRemoteFile(t *testing.T) {
 	for name, test := range cases {
 		test := test
 		t.Run(name, func(t *testing.T) {
-			require.Equal(t, test.valid, IsRemoteFile(test.url))
+			remote, _ := IsRemoteFile(test.url)
+			require.Equal(t, test.valid, remote)
 		})
 	}
 }

go.work.sum

@@ -1,8 +1,12 @@
 github.com/360EntSecGroup-Skylar/excelize v1.4.1/go.mod h1:vnax29X2usfl7HHkBrX5EvSCJcmH3dT9luvxzu8iGAE=
 github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg=
+github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
 github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/dustmop/soup v1.1.2-0.20190516214245-38228baa104e/go.mod h1:CgNC6SGbT+Xb8wGGvzilttZL1mc5sQ/5KkcxsZttMIk=
+github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk=
+github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84=
 github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/go-openapi/spec v0.19.3 h1:0XRyw8kguri6Yw4SxhsQA/atC88yqrk0+G4YhI2wabc=
@@ -16,6 +20,7 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
 github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
 github.com/markbates/pkger v0.17.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
@@ -35,6 +40,7 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t
 go.starlark.net v0.0.0-20190528202925-30ae18b8564f/go.mod h1:c1/X6cHgvdXj6pUlmWKMkuqRnW4K8x2vwt6JAaaircg=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs=
 golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
@@ -47,8 +53,10 @@ golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw=
 golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
 golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44=
 golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ=