Merge pull request #3022 from ordovicia/fix-3018

Author: hiddeco

pkg/cluster/kubernetes/resource/load_test.go

@@ -268,7 +268,7 @@ func debyte(r resource.Resource) resource.Resource {
 func TestLoadSome(t *testing.T) {
 	dir, cleanup := testfiles.TempDir(t)
 	defer cleanup()
-	if err := testfiles.WriteTestFiles(dir); err != nil {
+	if err := testfiles.WriteTestFiles(dir, testfiles.Files); err != nil {
 		t.Fatal(err)
 	}
 	objs, err := Load(dir, []string{dir}, false)
@@ -283,7 +283,7 @@ func TestLoadSome(t *testing.T) {
 func TestChartTracker(t *testing.T) {
 	dir, cleanup := testfiles.TempDir(t)
 	defer cleanup()
-	if err := testfiles.WriteTestFiles(dir); err != nil {
+	if err := testfiles.WriteTestFiles(dir, testfiles.Files); err != nil {
 		t.Fatal(err)
 	}
 
@@ -331,7 +331,7 @@ func TestChartTracker(t *testing.T) {
 func TestLoadSomeWithSopsNoneEncrypted(t *testing.T) {
 	dir, cleanup := testfiles.TempDir(t)
 	defer cleanup()
-	if err := testfiles.WriteTestFiles(dir); err != nil {
+	if err := testfiles.WriteTestFiles(dir, testfiles.Files); err != nil {
 		t.Fatal(err)
 	}
 	objs, err := Load(dir, []string{dir}, true)

pkg/cluster/kubernetes/testfiles/data.go

@@ -27,8 +27,8 @@ func TempDir(t *testing.T) (string, func()) {
 }
 
 // WriteTestFiles ... given a directory, create files in it, based on predetermined file content
-func WriteTestFiles(dir string) error {
-  return writeFiles(dir, Files)
+func WriteTestFiles(dir string, files map[string]string) error {
+	return writeFiles(dir, files)
 }
 
 // WriteSopsEncryptedTestFiles ... given a directory, create files in it, based on predetermined file content.
@@ -358,6 +358,43 @@ spec:
 `,
 }
 
+// -+- .flux.yaml
+//  +- base/    -+- kustomization.yaml
+//  |            +- foo.yaml
+//  +- staging/ -+- kustomization.yaml
+//               +- staging.yaml
+
+var FilesForKustomize = map[string]string{
+	".flux.yaml": `version: 1
+patchUpdated:
+  generators:
+  - command: kustomize build .
+  patchFile: flux-patch.yaml
+`,
+	"base/kustomization.yaml": `resources:
+- foo.yaml
+`,
+	"base/foo.yaml": `apiVersion: v1
+kind: Namespace
+metadata:
+  name: foo
+  annotations:
+    key: value
+`,
+	"staging/kustomization.yaml": `bases:
+- ../base/
+patches:
+- staging.yaml
+`,
+	"staging/staging.yaml": `apiVersion: v1
+kind: Namespace
+metadata:
+  name: foo
+  annotations:
+    env: staging
+`,
+}
+
 var SopsEncryptedFiles = map[string]string{
   "garbage": "This should just be ignored, since it is not YAML",
   "helloworld-deploy.yaml": `apiVersion: ENC[AES256_GCM,data:N/68Js00AtWIvks/pt+be5AW,iv:9Ke36D3faRNrMzm82Z9ETl3lOMhhWy8fh907K5e2Ar4=,tag:EfAzs1AQvLRH/tIQ+iZttw==,type:str]

pkg/cluster/kubernetes/testfiles/data_test.go

@@ -10,7 +10,7 @@ func TestWriteTestFiles(t *testing.T) {
 	dir, cleanup := TempDir(t)
 	defer cleanup()
 
-	if err := WriteTestFiles(dir); err != nil {
+	if err := WriteTestFiles(dir, Files); err != nil {
 		cleanup()
 		t.Fatal(err)
 	}

pkg/daemon/daemon_test.go

@@ -21,6 +21,7 @@ import (
 	"github.com/fluxcd/flux/pkg/api/v9"
 	"github.com/fluxcd/flux/pkg/cluster"
 	"github.com/fluxcd/flux/pkg/cluster/kubernetes"
+	"github.com/fluxcd/flux/pkg/cluster/kubernetes/testfiles"
 	"github.com/fluxcd/flux/pkg/cluster/mock"
 	"github.com/fluxcd/flux/pkg/event"
 	"github.com/fluxcd/flux/pkg/git"
@@ -664,7 +665,7 @@ func mockDaemon(t *testing.T) (*Daemon, func(), func(), *mock.Mock, *mockEventWr
 		},
 	}
 
-	repo, repoCleanup := gittest.Repo(t)
+	repo, repoCleanup := gittest.Repo(t, testfiles.Files)
 
 	syncTag := "flux-test"
 	params := git.Config{

pkg/daemon/sync.go

@@ -65,7 +65,7 @@ func (d *Daemon) Sync(ctx context.Context, started time.Time, newRevision string
 	}
 
 	// Retrieve change set of commits we need to sync
-	c, err := getChangeSet(ctx, ratchet, newRevision, d.Repo, d.GitTimeout, d.GitConfig.Paths)
+	c, err := d.getChangeSet(ctx, ratchet, newRevision)
 	if err != nil {
 		return err
 	}
@@ -127,7 +127,7 @@ func (d *Daemon) Sync(ctx context.Context, started time.Time, newRevision string
 
 // getChangeSet returns the change set of commits for this sync,
 // including the revision range and if it is an initial sync.
-func getChangeSet(ctx context.Context, state revisionRatchet, headRev string, repo *git.Repo, timeout time.Duration, paths []string) (changeSet, error) {
+func (d *Daemon) getChangeSet(ctx context.Context, state revisionRatchet, headRev string) (changeSet, error) {
 	var c changeSet
 	var err error
 
@@ -139,12 +139,17 @@ func getChangeSet(ctx context.Context, state revisionRatchet, headRev string, re
 	c.oldTagRev = currentRev
 	c.newTagRev = headRev
 
-	ctx, cancel := context.WithTimeout(ctx, timeout)
+	paths := d.GitConfig.Paths
+	if d.ManifestGenerationEnabled {
+		paths = []string{}
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, d.GitTimeout)
 	if c.oldTagRev != "" {
-		c.commits, err = repo.CommitsBetween(ctx, c.oldTagRev, c.newTagRev, false, paths...)
+		c.commits, err = d.Repo.CommitsBetween(ctx, c.oldTagRev, c.newTagRev, false, paths...)
 	} else {
 		c.initialSync = true
-		c.commits, err = repo.CommitsBefore(ctx, c.newTagRev, false, paths...)
+		c.commits, err = d.Repo.CommitsBefore(ctx, c.newTagRev, false, paths...)
 	}
 	cancel()
 

pkg/daemon/sync_test.go

@@ -41,8 +41,8 @@ var (
 	events *mockEventWriter
 )
 
-func daemon(t *testing.T) (*Daemon, func()) {
-	repo, repoCleanup := gittest.Repo(t)
+func daemon(t *testing.T, files map[string]string) (*Daemon, func()) {
+	repo, repoCleanup := gittest.Repo(t, files)
 
 	k8s = &mock.Mock{}
 	k8s.ExportFunc = func(ctx context.Context) ([]byte, error) { return nil, nil }
@@ -132,7 +132,7 @@ func checkSyncManifestsMetrics(t *testing.T, manifestSuccess, manifestFailures i
 }
 
 func TestPullAndSync_InitialSync(t *testing.T) {
-	d, cleanup := daemon(t)
+	d, cleanup := daemon(t, testfiles.Files)
 	defer cleanup()
 
 	syncCalled := 0
@@ -199,7 +199,7 @@ func TestPullAndSync_InitialSync(t *testing.T) {
 }
 
 func TestDoSync_NoNewCommits(t *testing.T) {
-	d, cleanup := daemon(t)
+	d, cleanup := daemon(t, testfiles.Files)
 	defer cleanup()
 
 	var syncTag = "syncity"
@@ -279,7 +279,7 @@ func TestDoSync_NoNewCommits(t *testing.T) {
 }
 
 func TestDoSync_WithNewCommit(t *testing.T) {
-	d, cleanup := daemon(t)
+	d, cleanup := daemon(t, testfiles.Files)
 	defer cleanup()
 
 	ctx := context.Background()
@@ -376,7 +376,7 @@ func TestDoSync_WithNewCommit(t *testing.T) {
 		t.Errorf("Sync was called with a nil syncDef")
 	}
 
-	// The emitted event has no workload ids
+	// An event is emitted and it only has a changed workload id
 	es, err := events.AllEvents(time.Time{}, -1, time.Time{})
 	if err != nil {
 		t.Error(err)
@@ -387,7 +387,6 @@ func TestDoSync_WithNewCommit(t *testing.T) {
 	} else {
 		gotResourceIDs := es[0].ServiceIDs
 		resource.IDs(gotResourceIDs).Sort()
-		// Event should only have changed workload ids
 		if !reflect.DeepEqual(gotResourceIDs, []resource.ID{resource.MustParseID("default:deployment/helloworld")}) {
 			t.Errorf("Unexpected event workload ids: %#v, expected: %#v", gotResourceIDs, []resource.ID{resource.MustParseID("default:deployment/helloworld")})
 		}
@@ -406,8 +405,125 @@ func TestDoSync_WithNewCommit(t *testing.T) {
 	}
 }
 
+func TestDoSync_WithKustomize(t *testing.T) {
+	d, cleanup := daemon(t, testfiles.FilesForKustomize)
+	defer cleanup()
+
+	d.GitConfig.Paths = []string{"staging"}
+	d.ManifestGenerationEnabled = true
+
+	ctx := context.Background()
+
+	var syncTag = "syncy-mcsyncface"
+	// Set the sync tag to head
+	var oldRevision, newRevision string
+	err := d.WithWorkingClone(ctx, func(checkout *git.Checkout) error {
+		ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+		defer cancel()
+
+		var err error
+		tagAction := git.TagAction{
+			Tag:      syncTag,
+			Revision: "master",
+			Message:  "Sync pointer",
+		}
+		err = checkout.MoveTagAndPush(ctx, tagAction)
+		if err != nil {
+			return err
+		}
+		oldRevision, err = checkout.HeadRevision(ctx)
+		if err != nil {
+			return err
+		}
+
+		// Push some new changes
+		absolutePath := path.Join(checkout.Dir(), "base", "foo.yaml")
+		def, err := ioutil.ReadFile(absolutePath)
+		if err != nil {
+			return err
+		}
+
+		newDef := bytes.Replace(def, []byte("key: value"), []byte("key: value2"), -1)
+		if err := ioutil.WriteFile(absolutePath, newDef, 0600); err != nil {
+			return err
+		}
+
+		commitAction := git.CommitAction{Author: "", Message: "test commit"}
+		err = checkout.CommitAndPush(ctx, commitAction, nil, true)
+		if err != nil {
+			return err
+		}
+		newRevision, err = checkout.HeadRevision(ctx)
+		return err
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = d.Repo.Refresh(ctx)
+	if err != nil {
+		t.Error(err)
+	}
+
+	syncCalled := 0
+	var syncDef *cluster.SyncSet
+	k8s.SyncFunc = func(def cluster.SyncSet) error {
+		syncCalled++
+		syncDef = &def
+		return nil
+	}
+
+	head, err := d.Repo.BranchHead(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	gitSync, _ := fluxsync.NewGitTagSyncProvider(d.Repo, syncTag, "", fluxsync.VerifySignaturesModeNone, d.GitConfig)
+	syncState := &lastKnownSyncState{logger: d.Logger, state: gitSync}
+
+	if err := d.Sync(ctx, time.Now().UTC(), head, syncState); err != nil {
+		t.Error(err)
+	}
+
+	// It applies everything
+	if syncCalled != 1 {
+		t.Errorf("Sync was not called once, was called %d times", syncCalled)
+	} else if syncDef == nil {
+		t.Errorf("Sync was called with a nil syncDef")
+	}
+
+	// An event is emitted and it only has a changed workload id
+	es, err := events.AllEvents(time.Time{}, -1, time.Time{})
+	if err != nil {
+		t.Error(err)
+	} else if len(es) != 1 {
+		t.Errorf("Unexpected events: %#v", es)
+	} else if es[0].Type != event.EventSync {
+		t.Errorf("Unexpected event type: %#v", es[0])
+	} else {
+		gotResourceIDs := es[0].ServiceIDs
+		resource.IDs(gotResourceIDs).Sort()
+		if !reflect.DeepEqual(gotResourceIDs, []resource.ID{resource.MustParseID("default:namespace/foo")}) {
+			t.Errorf("Unexpected event workload ids: %#v, expected: %#v", gotResourceIDs, []resource.ID{resource.MustParseID("default:namespace/foo")})
+		}
+	}
+
+	// It moves the tag
+	ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+	defer cancel()
+	if err := d.Repo.Refresh(ctx); err != nil {
+		t.Errorf("pulling sync tag: %v", err)
+	} else if revs, err := d.Repo.CommitsBetween(ctx, oldRevision, syncTag, false); err != nil {
+		t.Errorf("finding revisions before sync tag: %v", err)
+	} else if len(revs) <= 0 {
+		t.Errorf("Should have moved sync tag forward")
+	} else if revs[len(revs)-1].Revision != newRevision {
+		t.Errorf("Should have moved sync tag to HEAD (%s), but was moved to: %s", newRevision, revs[len(revs)-1].Revision)
+	}
+}
+
 func TestDoSync_WithErrors(t *testing.T) {
-	d, cleanup := daemon(t)
+	d, cleanup := daemon(t, testfiles.Files)
 	defer cleanup()
 
 	expectedResourceIDs := resource.IDs{}

pkg/git/gittest/repo.go

@@ -14,7 +14,7 @@ import (
 
 // Repo creates a new clone-able git repo, pre-populated with some kubernetes
 // files and a few commits. Also returns a cleanup func to clean up after.
-func Repo(t *testing.T) (*git.Repo, func()) {
+func Repo(t *testing.T, files map[string]string) (*git.Repo, func()) {
 	newDir, cleanup := testfiles.TempDir(t)
 
 	filesDir := filepath.Join(newDir, "files")
@@ -36,7 +36,8 @@ func Repo(t *testing.T) (*git.Repo, func()) {
 		cleanup()
 		t.Fatal(err)
 	}
-	if err = testfiles.WriteTestFiles(filesDir); err != nil {
+
+	if err = testfiles.WriteTestFiles(filesDir, files); err != nil {
 		cleanup()
 		t.Fatal(err)
 	}
@@ -79,7 +80,7 @@ func CheckoutWithConfig(t *testing.T, config git.Config, syncTag string) (*git.C
 	// This is to make sure that git commands don't have ambiguity problems between revisions and files.
 	testfiles.Files[config.Branch] = "Filename doctored to create a conflict with the git branch name"
 	testfiles.Files[syncTag] = "Filename doctored to create a conflict with the git sync tag"
-	repo, cleanup := Repo(t)
+	repo, cleanup := Repo(t, testfiles.Files)
 	if err := repo.Ready(context.Background()); err != nil {
 		cleanup()
 		t.Fatal(err)

pkg/git/gittest/repo_test.go

@@ -159,7 +159,7 @@ func TestSignedTag(t *testing.T) {
 }
 
 func TestCheckout(t *testing.T) {
-	repo, cleanup := Repo(t)
+	repo, cleanup := Repo(t, testfiles.Files)
 	defer cleanup()
 
 	sd, sg := make(chan struct{}), &sync.WaitGroup{}

pkg/git/operations_test.go

@@ -365,7 +365,7 @@ func createRepo(dir string, subdirs []string) error {
 			return err
 		}
 
-		if err = testfiles.WriteTestFiles(fullPath); err != nil {
+		if err = testfiles.WriteTestFiles(fullPath, testfiles.Files); err != nil {
 			return err
 		}
 		if err = execCommand("git", "-C", dir, "add", "--all"); err != nil {