implement dual-write workflows with go-workflows

the main drivers to replace with go-workflows are:
- we through the maintainer it's being used in production
  in a large scale system
- type-safety with generics, less boilerplate
- more approachable backend interface
- redis support

This commit conducted a refactoring of the code, added
unit tests, and addressed a corner-case in CheckKubeResource
method

.github/workflows/build-test.yaml

@@ -11,7 +11,7 @@ on: # yamllint disable-line rule:truthy
     types:
       - "checks_requested"
 env:
-  GO_VERSION: "~1.20.7"
+  GO_VERSION: "~1.21.1"
 jobs:
   paths-filter:
     runs-on: "ubuntu-latest"

Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=$BUILDPLATFORM golang:1.20-alpine3.18 AS builder
+FROM --platform=$BUILDPLATFORM golang:1.21.1-alpine3.18 AS builder
 ARG TARGETOS TARGETARCH
 
 WORKDIR /go/src/spicedb-kubeapi-proxy

e2e/go.mod

@@ -1,6 +1,8 @@
 module github.com/authzed/spicedb-kubeapi-proxy/e2e
 
-go 1.20
+go 1.21
+
+toolchain go1.21.0
 
 replace github.com/authzed/spicedb-kubeapi-proxy => ../
 
@@ -53,6 +55,7 @@ require (
 	github.com/coreos/go-semver v0.3.1 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/creasty/defaults v1.7.0 // indirect
+	github.com/cschleiden/go-workflows v0.16.2 // indirect
 	github.com/dalzilio/rudd v1.1.1-0.20230806153452-9e08a6ea8170 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dlmiddlecote/sqlstats v1.0.2 // indirect
@@ -69,6 +72,7 @@ require (
 	github.com/fatih/color v1.15.0 // indirect
 	github.com/felixge/httpsnoop v1.0.3 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/go-logr/logr v1.2.4 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-logr/zerologr v1.2.3 // indirect
@@ -111,23 +115,22 @@ require (
 	github.com/jackc/pgx-zerolog v0.0.0-20230315001418-f978528409eb // indirect
 	github.com/jackc/pgx/v5 v5.4.2 // indirect
 	github.com/jackc/puddle/v2 v2.2.1 // indirect
+	github.com/jellydator/ttlcache/v3 v3.0.0 // indirect
 	github.com/joho/godotenv v1.5.1 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/jzelinskie/cobrautil/v2 v2.0.0-20230714172849-80717639cec5 // indirect
 	github.com/jzelinskie/stringz v0.0.2 // indirect
-	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
 	github.com/klauspost/compress v1.16.7 // indirect
 	github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
 	github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
 	github.com/lthibault/jitterbug v2.0.0+incompatible // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/marusama/semaphore/v2 v2.5.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/mattn/go-sqlite3 v1.14.16 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
-	github.com/microsoft/durabletask-go v0.3.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/moby/sys/mountinfo v0.6.2 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@@ -147,7 +150,6 @@ require (
 	github.com/prometheus/client_model v0.4.0 // indirect
 	github.com/prometheus/common v0.44.0 // indirect
 	github.com/prometheus/procfs v0.11.0 // indirect
-	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/rs/cors v1.9.0 // indirect
 	github.com/rs/zerolog v1.29.1 // indirect
 	github.com/scylladb/go-set v1.0.2 // indirect
@@ -159,6 +161,7 @@ require (
 	github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace // indirect
 	github.com/spf13/viper v1.16.0 // indirect
 	github.com/stoewer/go-strcase v1.3.0 // indirect
+	github.com/stretchr/objx v0.5.0 // indirect
 	github.com/stretchr/testify v1.8.4 // indirect
 	github.com/subosito/gotenv v1.4.2 // indirect
 	go.etcd.io/etcd/api/v3 v3.5.9 // indirect
@@ -221,16 +224,6 @@ require (
 	k8s.io/mount-utils v0.0.0 // indirect
 	k8s.io/pod-security-admission v0.0.0 // indirect
 	k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect
-	lukechampine.com/uint128 v1.2.0 // indirect
-	modernc.org/cc/v3 v3.40.0 // indirect
-	modernc.org/ccgo/v3 v3.16.13 // indirect
-	modernc.org/libc v1.22.5 // indirect
-	modernc.org/mathutil v1.5.0 // indirect
-	modernc.org/memory v1.5.0 // indirect
-	modernc.org/opt v0.1.3 // indirect
-	modernc.org/sqlite v1.22.1 // indirect
-	modernc.org/strutil v1.1.3 // indirect
-	modernc.org/token v1.0.1 // indirect
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect

e2e/proxy_test.go

@@ -113,7 +113,7 @@ var _ = Describe("Proxy", func() {
 				Expect(chaniList).To(ContainElement(chaniNamespace))
 			})
 
-			It("cleans up dual writes on kube failures", func(ctx context.Context) {
+			It("recovers when there are kube failures", func(ctx context.Context) {
 				// paul creates his namespace
 				Expect(CreateNamespace(ctx, paulClient, paulNamespace)).To(Succeed())
 
@@ -125,21 +125,22 @@ var _ = Describe("Proxy", func() {
 				} else {
 					failpoints.EnableFailPoint("panicKubeWrite", 1)
 				}
-				Expect(CreateNamespace(ctx, chaniClient, chaniNamespace)).ToNot(BeNil())
+				// Chani's write panics, but is retried
+				Expect(CreateNamespace(ctx, chaniClient, chaniNamespace)).To(Succeed())
 
-				// paul creates chani's namespace
-				Expect(CreateNamespace(ctx, paulClient, chaniNamespace)).To(Succeed())
+				// paul isn't able to create chanis namespace
+				Expect(CreateNamespace(ctx, paulClient, chaniNamespace)).ToNot(BeNil())
 
-				// paul can get both namespaces
+				// paul can only get his namespace
 				Expect(GetNamespace(ctx, paulClient, paulNamespace)).To(Succeed())
-				Expect(GetNamespace(ctx, paulClient, chaniNamespace)).To(Succeed())
+				Expect(GetNamespace(ctx, paulClient, chaniNamespace)).ToNot(BeNil())
 
-				// chani can't get her namespace - this indicates the spicedb write was rolled back
-				// from the failed dual write above
-				Expect(k8serrors.IsNotFound(GetNamespace(ctx, chaniClient, chaniNamespace))).To(BeTrue())
+				// chani can get her namespace - this indicates the workflow was retried and eventually succeeded
+				Expect(GetNamespace(ctx, chaniClient, paulNamespace)).ToNot(BeNil())
+				Expect(GetNamespace(ctx, chaniClient, chaniNamespace)).To(Succeed())
 			})
 
-			It("recovers dual writes when kube write succeeds but crashes", func(ctx context.Context) {
+			It("recovers when kube write succeeds but crashes", func(ctx context.Context) {
 				// paul creates his namespace
 				Expect(CreateNamespace(ctx, paulClient, paulNamespace)).To(Succeed())
 
@@ -179,34 +180,30 @@ var _ = Describe("Proxy", func() {
 				Expect(k8serrors.IsNotFound(GetNamespace(ctx, chaniClient, chaniNamespace))).To(BeTrue())
 			})
 
-			It("recovers dual writes when there are spicedb write failures", func(ctx context.Context) {
+			It("recovers when there are spicedb write failures", func(ctx context.Context) {
 				// paul creates his namespace
 				Expect(CreateNamespace(ctx, paulClient, paulNamespace)).To(Succeed())
 
-				// make spicedb write crash on chani's namespace write
+				// make spicedb write crash on chani's namespace write, eventually succeeds
 				failpoints.EnableFailPoint("panicWriteSpiceDB", 1)
-				Expect(CreateNamespace(ctx, chaniClient, chaniNamespace)).ToNot(BeNil())
+				Expect(CreateNamespace(ctx, chaniClient, chaniNamespace)).To(Succeed())
 
-				// paul creates chani's namespace so that the namespace exists
-				Expect(CreateNamespace(ctx, paulClient, chaniNamespace)).To(Succeed())
+				// paul is unable to create chani's namespace as it's already claimed
+				Expect(CreateNamespace(ctx, paulClient, chaniNamespace)).ToNot(BeNil())
 
-				// check that chani can't get her namespace, indirectly showing
-				// that the spicedb write was rolled back
-				Expect(k8serrors.IsNotFound(GetNamespace(ctx, chaniClient, chaniNamespace))).To(BeTrue())
+				// Check Chani is able to get namespace
+				Expect(GetNamespace(ctx, chaniClient, chaniNamespace)).To(Succeed())
 
-				// confirm the relationship doesn't exist
+				// confirm the relationship exists
 				Expect(len(GetAllTuples(ctx, &v1.RelationshipFilter{
 					ResourceType:          "namespace",
 					OptionalResourceId:    chaniNamespace,
 					OptionalRelation:      "creator",
 					OptionalSubjectFilter: &v1.SubjectFilter{SubjectType: "user", OptionalSubjectId: "chani"},
-				}))).To(BeZero())
-
-				// confirm paul can get the namespace
-				Expect(GetNamespace(ctx, paulClient, chaniNamespace)).To(Succeed())
+				}))).ToNot(BeZero())
 			})
 
-			It("recovers dual writes when spicedb write succeeds but crashes", func(ctx context.Context) {
+			It("recovers when spicedb write succeeds but crashes", func(ctx context.Context) {
 				// paul creates his namespace
 				Expect(CreateNamespace(ctx, paulClient, paulNamespace)).To(Succeed())
 

go.mod

@@ -1,15 +1,17 @@
 module github.com/authzed/spicedb-kubeapi-proxy
 
-go 1.20
+go 1.21
+
+toolchain go1.21.0
 
 require (
 	github.com/authzed/authzed-go v0.9.1-0.20230810180432-2fb0fd4c66dd
 	github.com/authzed/grpcutil v0.0.0-20230703173955-bdd0ac3f16a5
 	github.com/authzed/spicedb v1.24.1-0.20230821163419-e4bb3adfd50b
 	github.com/cespare/xxhash/v2 v2.2.0
+	github.com/cschleiden/go-workflows v0.16.2
 	github.com/dustin/go-humanize v1.0.1
 	github.com/google/uuid v1.3.0
-	github.com/microsoft/durabletask-go v0.3.0
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace
 	github.com/stretchr/testify v1.8.4
@@ -22,6 +24,8 @@ require (
 	k8s.io/kubernetes v1.28.0
 )
 
+replace github.com/cschleiden/go-workflows => github.com/vroldanbet/go-workflows v0.0.0-20230918124723-368160933470
+
 require (
 	buf.build/gen/go/gogo/protobuf/protocolbuffers/go v1.31.0-20210810001428-4df00b267f94.1 // indirect
 	github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect
@@ -76,6 +80,7 @@ require (
 	github.com/fatih/color v1.15.0 // indirect
 	github.com/felixge/httpsnoop v1.0.3 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/go-logr/logr v1.2.4 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-logr/zapr v1.2.4 // indirect
@@ -114,6 +119,7 @@ require (
 	github.com/jackc/pgx-zerolog v0.0.0-20230315001418-f978528409eb // indirect
 	github.com/jackc/pgx/v5 v5.4.2 // indirect
 	github.com/jackc/puddle/v2 v2.2.1 // indirect
+	github.com/jellydator/ttlcache/v3 v3.0.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/jzelinskie/cobrautil/v2 v2.0.0-20230714172849-80717639cec5 // indirect
@@ -125,7 +131,6 @@ require (
 	github.com/lthibault/jitterbug v2.0.0+incompatible // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/marusama/semaphore/v2 v2.5.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
@@ -160,6 +165,7 @@ require (
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/viper v1.16.0 // indirect
 	github.com/stoewer/go-strcase v1.3.0 // indirect
+	github.com/stretchr/objx v0.5.0 // indirect
 	github.com/subosito/gotenv v1.4.2 // indirect
 	go.etcd.io/etcd/api/v3 v3.5.9 // indirect
 	go.etcd.io/etcd/client/pkg/v3 v3.5.9 // indirect
@@ -217,11 +223,11 @@ require (
 	lukechampine.com/uint128 v1.2.0 // indirect
 	modernc.org/cc/v3 v3.40.0 // indirect
 	modernc.org/ccgo/v3 v3.16.13 // indirect
-	modernc.org/libc v1.22.5 // indirect
+	modernc.org/libc v1.24.1 // indirect
 	modernc.org/mathutil v1.5.0 // indirect
-	modernc.org/memory v1.5.0 // indirect
+	modernc.org/memory v1.6.0 // indirect
 	modernc.org/opt v0.1.3 // indirect
-	modernc.org/sqlite v1.22.1 // indirect
+	modernc.org/sqlite v1.25.0 // indirect
 	modernc.org/strutil v1.1.3 // indirect
 	modernc.org/token v1.0.1 // indirect
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 // indirect