Skip to content

Commit

Permalink
feat: Added feast Go operator db stores support (#4771)
Browse files Browse the repository at this point in the history
* Add support for db stores in feast go operator

Signed-off-by: Theodor Mihalache <tmihalac@redhat.com>

* Added CR example for store persistence

Signed-off-by: Theodor Mihalache <tmihalac@redhat.com>

* Fixed incorrect yaml tag in RegistryConfig struct

Signed-off-by: Theodor Mihalache <tmihalac@redhat.com>

* Removed leftovers comments from hasAttrib function

Signed-off-by: Theodor Mihalache <tmihalac@redhat.com>

* Added another check that object parameter type is the same as value type in hasAttrib

Signed-off-by: Theodor Mihalache <tmihalac@redhat.com>

* Reverted latest commit

Signed-off-by: Theodor Mihalache <tmihalac@redhat.com>

---------

Signed-off-by: Theodor Mihalache <tmihalac@redhat.com>
  • Loading branch information
tmihalac authored Nov 26, 2024
1 parent d2ba4ef commit 3302363
Show file tree
Hide file tree
Showing 16 changed files with 1,176 additions and 163 deletions.
66 changes: 62 additions & 4 deletions infra/feast-operator/api/v1alpha1/featurestore_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,11 +75,13 @@ type OfflineStore struct {
}

// OfflineStorePersistence configures the persistence settings for the offline store service
// +kubebuilder:validation:XValidation:rule="[has(self.file), has(self.store)].exists_one(c, c)",message="One selection required between file or store."
type OfflineStorePersistence struct {
FilePersistence *OfflineStoreFilePersistence `json:"file,omitempty"`
FilePersistence *OfflineStoreFilePersistence `json:"file,omitempty"`
DBPersistence *OfflineStoreDBStorePersistence `json:"store,omitempty"`
}

// OfflineStorePersistence configures the file-based persistence for the offline store service
// OfflineStoreFilePersistence configures the file-based persistence for the offline store service
type OfflineStoreFilePersistence struct {
// +kubebuilder:validation:Enum=dask;duckdb
Type string `json:"type,omitempty"`
Expand All @@ -91,15 +93,35 @@ var ValidOfflineStoreFilePersistenceTypes = []string{
"duckdb",
}

// OfflineStoreDBStorePersistence configures the DB store persistence for the offline store service
type OfflineStoreDBStorePersistence struct {
// +kubebuilder:validation:Enum=snowflake.offline;bigquery;redshift;spark;postgres;feast_trino.trino.TrinoOfflineStore;redis
Type string `json:"type,omitempty"`
SecretRef *corev1.LocalObjectReference `json:"secretRef,omitempty"`
SecretKeyName string `json:"secretKeyName,omitempty"`
}

var ValidOfflineStoreDBStorePersistenceTypes = []string{
"snowflake.offline",
"bigquery",
"redshift",
"spark",
"postgres",
"feast_trino.trino.TrinoOfflineStore",
"redis",
}

// OnlineStore configures the deployed online store service
type OnlineStore struct {
ServiceConfigs `json:",inline"`
Persistence *OnlineStorePersistence `json:"persistence,omitempty"`
}

// OnlineStorePersistence configures the persistence settings for the online store service
// +kubebuilder:validation:XValidation:rule="[has(self.file), has(self.store)].exists_one(c, c)",message="One selection required between file or store."
type OnlineStorePersistence struct {
FilePersistence *OnlineStoreFilePersistence `json:"file,omitempty"`
FilePersistence *OnlineStoreFilePersistence `json:"file,omitempty"`
DBPersistence *OnlineStoreDBStorePersistence `json:"store,omitempty"`
}

// OnlineStoreFilePersistence configures the file-based persistence for the offline store service
Expand All @@ -111,6 +133,28 @@ type OnlineStoreFilePersistence struct {
PvcConfig *PvcConfig `json:"pvc,omitempty"`
}

// OnlineStoreDBStorePersistence configures the DB store persistence for the offline store service
type OnlineStoreDBStorePersistence struct {
// +kubebuilder:validation:Enum=snowflake.online;redis;ikv;datastore;dynamodb;bigtable;postgres;cassandra;mysql;hazelcast;singlestore
Type string `json:"type,omitempty"`
SecretRef *corev1.LocalObjectReference `json:"secretRef,omitempty"`
SecretKeyName string `json:"secretKeyName,omitempty"`
}

var ValidOnlineStoreDBStorePersistenceTypes = []string{
"snowflake.online",
"redis",
"ikv",
"datastore",
"dynamodb",
"bigtable",
"postgres",
"cassandra",
"mysql",
"hazelcast",
"singlestore",
}

// LocalRegistryConfig configures the deployed registry service
type LocalRegistryConfig struct {
ServiceConfigs `json:",inline"`
Expand All @@ -119,7 +163,8 @@ type LocalRegistryConfig struct {

// RegistryPersistence configures the persistence settings for the registry service
type RegistryPersistence struct {
FilePersistence *RegistryFilePersistence `json:"file,omitempty"`
FilePersistence *RegistryFilePersistence `json:"file,omitempty"`
DBPersistence *RegistryDBStorePersistence `json:"store,omitempty"`
}

// RegistryFilePersistence configures the file-based persistence for the registry service
Expand All @@ -133,6 +178,19 @@ type RegistryFilePersistence struct {
S3AdditionalKwargs *map[string]string `json:"s3_additional_kwargs,omitempty"`
}

// RegistryDBStorePersistence configures the DB store persistence for the registry service
type RegistryDBStorePersistence struct {
// +kubebuilder:validation:Enum=sql;snowflake.registry
Type string `json:"type,omitempty"`
SecretRef *corev1.LocalObjectReference `json:"secretRef,omitempty"`
SecretKeyName string `json:"secretKeyName,omitempty"`
}

var ValidRegistryDBStorePersistenceTypes = []string{
"snowflake.registry",
"sql",
}

// PvcConfig defines the settings for a persistent file store based on PVCs.
// We can refer to an existing PVC using the `Ref` field, or create a new one using the `Create` field.
// +kubebuilder:validation:XValidation:rule="[has(self.ref), has(self.create)].exists_one(c, c)",message="One selection is required between ref and create."
Expand Down
75 changes: 75 additions & 0 deletions infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 3302363

Please sign in to comment.