diff --git a/.github/workflows/test-gh.yml b/.github/workflows/test-gh.yml
index 1da1476748..dcaee19793 100644
--- a/.github/workflows/test-gh.yml
+++ b/.github/workflows/test-gh.yml
@@ -43,6 +43,8 @@ jobs:
         mkdir /tmp/bin
         export PATH=/tmp/bin:$PATH
 
+        ./hack/verify-no-dirty-files.sh
+
         wget -O- https://github.com/kubernetes/minikube/releases/download/v1.10.0/minikube-linux-amd64 > /tmp/bin/minikube
         echo "9d34cb50bc39f80d39f92d1fb7cb23a271504b519f5e805574894d395ce3e7b3  /tmp/bin/minikube" | sha256sum -c -
         chmod +x /tmp/bin/minikube
diff --git a/hack/build.sh b/hack/build.sh
index 1bd88ce90c..3096ee4d0d 100755
--- a/hack/build.sh
+++ b/hack/build.sh
@@ -13,6 +13,7 @@ go mod tidy
 go build $repro_flags -mod=vendor -o controller ./cmd/main.go
 ls -la ./controller
 
+./hack/gen-crds.sh
 ytt -f config/ >/dev/null
 
 echo SUCCESS
diff --git a/config/crd-overlay.yml b/hack/crd-overlay.yml
similarity index 91%
rename from config/crd-overlay.yml
rename to hack/crd-overlay.yml
index e21c5ad73e..43c77d19f8 100644
--- a/config/crd-overlay.yml
+++ b/hack/crd-overlay.yml
@@ -19,7 +19,7 @@ spec:
 #@overlay/match missing_ok=True
 status:
 
-#@overlay/match by=overlay.subset({"metadata":{"name":"internalpackagemetadata.internal.packaging.carvel.dev"}}), expects="0+"
+#@overlay/match by=overlay.subset({"metadata":{"name":"internalpackagemetadata.internal.packaging.carvel.dev"}})
 ---
 metadata:
   name: internalpackagemetadatas.internal.packaging.carvel.dev
diff --git a/hack/deploy-test.sh b/hack/deploy-test.sh
index 11d7b1ae57..9ef7f407d8 100755
--- a/hack/deploy-test.sh
+++ b/hack/deploy-test.sh
@@ -3,4 +3,3 @@
 set -e
 
 ./hack/build.sh && ytt -f config/ -f config-test/ | kbld -f- | kapp deploy -a kc -f- -c -y
-
diff --git a/hack/gen-crds.sh b/hack/gen-crds.sh
index 0655e6e435..a614a85ce2 100755
--- a/hack/gen-crds.sh
+++ b/hack/gen-crds.sh
@@ -11,6 +11,6 @@ go run ./vendor/sigs.k8s.io/controller-tools/cmd/controller-gen/main.go \
   output:dir=./tmp/crds \
   paths=./pkg/apis/...
 
-ytt -f tmp/crds -f config/crd-overlay.yml > config/crds.yml
+ytt -f tmp/crds -f ./hack/crd-overlay.yml > config/crds.yml
 
 rm -rf tmp/crds
diff --git a/hack/verify-no-dirty-files.sh b/hack/verify-no-dirty-files.sh
new file mode 100755
index 0000000000..476a971fb5
--- /dev/null
+++ b/hack/verify-no-dirty-files.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -e
+
+./hack/build.sh
+
+if ! git diff --exit-code >/dev/null; then
+  echo "Error: Running ./hack/build.sh resulted in non zero exit code from git diff. Please run './hack/build.sh' and 'git add' the generated file(s)."
+  exit 1
+fi