Skip to content

Commit 1bd1296

Browse files
FxKuFxKu
committed
provide event stream API
completed codegen manually provide update and compare code for fes add db check for specified databases and tables add unit test
1 parent 2057ad8 commit 1bd1296

33 files changed

+1779
-14
lines changed

‎hack/update-codegen.sh‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,18 @@
11
#!/usr/bin/env bash
2+
set -eou pipefail
23

3-
set -o errexit
4-
set -o nounset
5-
set -o pipefail
4+
GOPKG="github.com/zalando/postgres-operator"
5+
SCRIPT_ROOT="$(dirname "${BASH_SOURCE[0]}")/.."
66

7-
SCRIPT_ROOT=$(dirname ${BASH_SOURCE})/..
8-
CODEGEN_PKG=${CODEGEN_PKG:-$(cd ${SCRIPT_ROOT}; ls -d -1 ./vendor/k8s.io/code-generator 2>/dev/null || echo ${GOPATH}/src/k8s.io/code-generator)}
7+
rm -rf "${SCRIPT_ROOT}/generated"
98

10-
bash "${CODEGEN_PKG}/generate-groups.sh" all \
11-
github.com/zalando/postgres-operator/pkg/generated github.com/zalando/postgres-operator/pkg/apis \
12-
"acid.zalan.do:v1" \
13-
--go-header-file "${SCRIPT_ROOT}"/hack/custom-boilerplate.go.txt
9+
go run k8s.io/code-generator/cmd/deepcopy-gen \
10+
--input-dirs ${GOPKG}/pkg/apis/acid.zalan.do/v1,${GOPKG}/pkg/apis/zalando.org/v1alpha1 \
11+
-O zz_generated.deepcopy \
12+
--bounding-dirs ${GOPKG}/pkg/apis \
13+
--go-header-file "${SCRIPT_ROOT}/hack/custom-boilerplate.go.txt" \
14+
-o "${SCRIPT_ROOT}/generated"
15+
16+
cp -rv "${SCRIPT_ROOT}/generated/${GOPKG}"/* .
17+
18+
rm -rf "${SCRIPT_ROOT}/generated"

‎manifests/operator-service-account-rbac.yaml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,20 @@ rules:
3535
- get
3636
- list
3737
- watch
38+
# all verbs allowed for event streams
39+
- apiGroups:
40+
- zalando.org
41+
resources:
42+
- fabriceventstream
43+
verbs:
44+
- create
45+
- delete
46+
- deletecollection
47+
- get
48+
- list
49+
- patch
50+
- update
51+
- watch
3852
# to create or get/update CRDs when starting up
3953
- apiGroups:
4054
- apiextensions.k8s.io

‎pkg/apis/acid.zalan.do/v1/crds.go‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -657,6 +657,53 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{
657657
},
658658
},
659659
},
660+
"streams": {
661+
Type: "array",
662+
Items: &apiextv1.JSONSchemaPropsOrArray{
663+
Schema: &apiextv1.JSONSchemaProps{
664+
Type: "object",
665+
Required: []string{"type"},
666+
Properties: map[string]apiextv1.JSONSchemaProps{
667+
"database": {
668+
Type: "string",
669+
},
670+
"filter": {
671+
Type: "object",
672+
AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{
673+
Schema: &apiextv1.JSONSchemaProps{
674+
Type: "string",
675+
},
676+
},
677+
},
678+
"sqsArn": {
679+
Type: "string",
680+
},
681+
"tables": {
682+
Type: "object",
683+
AdditionalProperties: &apiextv1.JSONSchemaPropsOrBool{
684+
Schema: &apiextv1.JSONSchemaProps{
685+
Type: "string",
686+
},
687+
},
688+
},
689+
"type": {
690+
Type: "string",
691+
Enum: []apiextv1.JSON{
692+
{
693+
Raw: []byte(`"nakadi"`),
694+
},
695+
{
696+
Raw: []byte(`"sqs"`),
697+
},
698+
{
699+
Raw: []byte(`"wal"`),
700+
},
701+
},
702+
},
703+
},
704+
},
705+
},
706+
},
660707
"teamId": {
661708
Type: "string",
662709
},

‎pkg/apis/acid.zalan.do/v1/postgresql_type.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ type PostgresSpec struct {
7474
ServiceAnnotations map[string]string `json:"serviceAnnotations,omitempty"`
7575
TLS *TLSDescription `json:"tls,omitempty"`
7676
AdditionalVolumes []AdditionalVolume `json:"additionalVolumes,omitempty"`
77+
Streams []Stream `json:"stream,omitempty"`
7778

7879
// deprecated json tags
7980
InitContainersOld []v1.Container `json:"init_containers,omitempty"`
@@ -224,3 +225,14 @@ type ConnectionPooler struct {
224225

225226
Resources `json:"resources,omitempty"`
226227
}
228+
229+
type Stream struct {
230+
Type string `json:"type"`
231+
Database string `json:"database,omitempty"`
232+
Tables map[string]string `json:"tables,omitempty"`
233+
Filter map[string]string `json:"filter,omitempty"`
234+
BatchSize uint32 `json:"batchSize,omitempty"`
235+
SqsArn string `json:"sqsArn,omitempty"`
236+
QueueName string `json:"queueName,omitempty"`
237+
User string `json:"user,omitempty"`
238+
}

‎pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go‎

Lines changed: 37 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/apis/zalando.org/register.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
package zalando
2+
3+
const (
4+
// GroupName is the group name for the operator CRDs
5+
GroupName = "zalando.org"
6+
)

‎pkg/apis/zalando.org/v1alpha1/fabriceventstream.go‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
package v1alpha1
2+
3+
import (
4+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
5+
)
6+
7+
// +genclient
8+
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
9+
10+
// FabricEventStream defines FabricEventStream Custom Resource Definition Object.
11+
type FabricEventStream struct {
12+
metav1.TypeMeta `json:",inline"`
13+
metav1.ObjectMeta `json:"metadata,omitempty"`
14+
15+
Spec FabricEventStreamSpec `json:"spec"`
16+
}
17+
18+
// FabricEventStreamSpec defines the specification for the FabricEventStream TPR.
19+
type FabricEventStreamSpec struct {
20+
ApplicationId string `json:"applicationId"`
21+
EventStreams []EventStream `json:"eventStreams"`
22+
}
23+
24+
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
25+
26+
// FabricEventStreamList defines a list of FabricEventStreams .
27+
type FabricEventStreamList struct {
28+
metav1.TypeMeta `json:",inline"`
29+
metav1.ListMeta `json:"metadata"`
30+
31+
Items []FabricEventStream `json:"items"`
32+
}
33+
34+
// EventStream defines the source, flow and sink of the event stream
35+
type EventStream struct {
36+
EventStreamFlow EventStreamFlow `json:"flow"`
37+
EventStreamSink EventStreamSink `json:"sink"`
38+
EventStreamSource EventStreamSource `json:"source"`
39+
}
40+
41+
// EventStreamFlow defines the flow characteristics of the event stream
42+
type EventStreamFlow struct {
43+
Type string `json:"type"`
44+
DataTypeColumn string `json:"dataTypeColumn,omitempty"`
45+
DataOpColumn string `json:"dataOpColumn,omitempty"`
46+
MetadataColumn string `json:"metadataColumn,omitempty"`
47+
DataColumn string `json:"dataColumn,omitempty"`
48+
CallHomeIdColumn string `json:"callHomeIdColumn,omitempty"`
49+
CallHomeUrl string `json:"callHomeUrl,omitempty"`
50+
}
51+
52+
// EventStreamSink defines the target of the event stream
53+
type EventStreamSink struct {
54+
Type string `json:"type"`
55+
EventType string `json:"eventType,omitempty"`
56+
MaxBatchSize uint32 `json:"maxBatchSize,omitempty"`
57+
QueueName string `json:"queueName,omitempty"`
58+
}
59+
60+
// EventStreamSource defines the source of the event stream and connection for FES operator
61+
type EventStreamSource struct {
62+
Type string `json:"type"`
63+
Schema string `json:"schema,omitempty" defaults:"public"`
64+
EventStreamTable EventStreamTable `json:"table"`
65+
Filter string `json:"filter,omitempty"`
66+
Connection Connection `json:"jdbcConnection"`
67+
}
68+
69+
// EventStreamTable defines the name and ID column to be used for streaming
70+
type EventStreamTable struct {
71+
Name string `json:"name"`
72+
IDColumn string `json:"idColumn,omitempty" defaults:"id"`
73+
}
74+
75+
// Connection to be used for allowing the FES operator to connect to a database
76+
type Connection struct {
77+
Url string `json:"jdbcUrl"`
78+
SlotName string `json:"slotName"`
79+
DBAuth DBAuth `json:"databaseAuthentication"`
80+
}
81+
82+
// DBAuth specifies the credentials to be used for connecting with the database
83+
type DBAuth struct {
84+
Type string `json:"type"`
85+
Name string `json:"name,omitempty"`
86+
UserKey string `json:"userKey,omitempty"`
87+
PasswordKey string `json:"passwordKey,omitempty"`
88+
}

‎pkg/apis/zalando.org/v1alpha1/register.go‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
package v1alpha1
2+
3+
import (
4+
"github.com/zalando/postgres-operator/pkg/apis/zalando.org"
5+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
6+
"k8s.io/apimachinery/pkg/runtime"
7+
"k8s.io/apimachinery/pkg/runtime/schema"
8+
"k8s.io/client-go/kubernetes/scheme"
9+
)
10+
11+
// APIVersion of the `fabriceventstream` CRD
12+
const (
13+
APIVersion = "v1alpha1"
14+
)
15+
16+
var (
17+
schemeBuilder = runtime.NewSchemeBuilder(addKnownTypes)
18+
// AddToScheme applies all the stored functions to the scheme. A non-nil error
19+
// indicates that one function failed and the attempt was abandoned.
20+
AddToScheme = schemeBuilder.AddToScheme
21+
)
22+
23+
func init() {
24+
err := AddToScheme(scheme.Scheme)
25+
if err != nil {
26+
panic(err)
27+
}
28+
}
29+
30+
// SchemeGroupVersion is the group version used to register these objects.
31+
var SchemeGroupVersion = schema.GroupVersion{Group: zalando.GroupName, Version: APIVersion}
32+
33+
// Resource takes an unqualified resource and returns a Group-qualified GroupResource.
34+
func Resource(resource string) schema.GroupResource {
35+
return SchemeGroupVersion.WithResource(resource).GroupResource()
36+
}
37+
38+
// addKnownTypes adds the set of types defined in this package to the supplied scheme.
39+
func addKnownTypes(scheme *runtime.Scheme) error {
40+
scheme.AddKnownTypes(SchemeGroupVersion,
41+
&FabricEventStream{},
42+
&FabricEventStreamList{},
43+
)
44+
metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
45+
return nil
46+
}

0 commit comments

Comments
 (0)