From 44edc0d4507ff04931fc846ae8a96970346a83d1 Mon Sep 17 00:00:00 2001 From: Leon Date: Wed, 14 Aug 2024 13:09:42 +0800 Subject: [PATCH] chore: integrate kb-agent with controller (#7821) Co-authored-by: Ursasi --- Makefile | 6 +- .../v1alpha1/componentdefinition_types.go | 45 +- cmd/{kb_agent => kbagent}/main.go | 40 +- ...ps.kubeblocks.io_componentdefinitions.yaml | 55 +-- controllers/apps/component_controller_test.go | 57 ++- controllers/apps/component_utils.go | 18 - .../apps/opsrequest_controller_test.go | 62 +-- ...transformer_component_account_provision.go | 46 +-- .../apps/transformer_component_hostnetwork.go | 102 +---- .../apps/transformer_component_rbac.go | 4 - .../apps/transformer_component_workload.go | 38 +- ...ps.kubeblocks.io_componentdefinitions.yaml | 55 +-- docker/Dockerfile-tools | 27 +- docs/developer_docs/api-reference/cluster.md | 45 +- pkg/controller/component/its_convertor.go | 35 +- .../component/its_convertor_test.go | 61 --- pkg/controller/component/kbagent.go | 389 ++++++++++++++++++ pkg/controller/component/lifecycle/errors.go | 42 ++ pkg/controller/component/lifecycle/kbagent.go | 235 +++++++++++ .../component/lifecycle/lfa_account.go} | 28 +- .../component/lifecycle/lfa_common.go} | 12 +- .../component/lifecycle/lfa_component.go} | 47 ++- .../component/lifecycle/lfa_data.go} | 40 +- .../component/lifecycle/lfa_member.go | 80 ++++ .../component/lifecycle/lfa_role.go | 41 ++ .../component/lifecycle/lifecycle.go | 81 ++++ .../component/synthesize_component.go | 6 +- pkg/controller/component/vars.go | 8 +- pkg/controller/component/workload_utils.go | 9 + pkg/controller/factory/builder_test.go | 2 +- .../instanceset/pod_role_event_handler.go | 31 ++ pkg/controllerutil/pod_utils.go | 13 + pkg/kb_agent/cronjobs/checkrole.go | 93 ----- pkg/kb_agent/cronjobs/checkrole_test.go | 133 ------ pkg/kb_agent/cronjobs/job.go | 106 ----- pkg/kb_agent/cronjobs/job_test.go | 99 ----- pkg/kb_agent/cronjobs/manager.go | 58 --- pkg/kb_agent/cronjobs/manager_test.go | 56 --- pkg/kb_agent/handlers/action.go | 117 ------ pkg/kb_agent/handlers/action_test.go | 123 ------ pkg/kb_agent/handlers/exec_handler.go | 77 ---- pkg/kb_agent/handlers/exec_handler_test.go | 103 ----- pkg/kb_agent/handlers/grpc_handler_test.go | 60 --- pkg/kb_agent/httpserver/apis.go | 123 ------ pkg/kb_agent/httpserver/apis_test.go | 129 ------ pkg/kb_agent/httpserver/config.go | 50 --- pkg/kb_agent/httpserver/endpoint.go | 47 --- pkg/kb_agent/httpserver/server.go | 161 -------- pkg/kb_agent/httpserver/server_test.go | 100 ----- pkg/kb_agent/util/command.go | 62 --- pkg/kb_agent/util/event_test.go | 63 --- pkg/kb_agent/util/types.go | 66 --- pkg/kbagent/client/client.go | 98 +++++ pkg/kbagent/client/client_mock.go | 72 ++++ .../client/generate.go} | 17 +- pkg/kbagent/client/httpclient.go | 105 +++++ pkg/kbagent/proto/proto.go | 68 +++ pkg/kbagent/server/httpserver.go | 235 +++++++++++ pkg/kbagent/server/types.go | 51 +++ pkg/kbagent/service/action.go | 132 ++++++ pkg/kbagent/service/command.go | 212 ++++++++++ pkg/kbagent/service/probe.go | 194 +++++++++ pkg/kbagent/service/service.go | 62 +++ pkg/kbagent/setup.go | 104 +++++ .../k8s_client.go => kbagent/util/env.go} | 32 +- pkg/{kb_agent => kbagent}/util/event.go | 80 ++-- pkg/testutil/apps/constant.go | 15 +- 67 files changed, 2576 insertions(+), 2487 deletions(-) rename cmd/{kb_agent => kbagent}/main.go (61%) delete mode 100644 pkg/controller/component/its_convertor_test.go create mode 100644 pkg/controller/component/kbagent.go create mode 100644 pkg/controller/component/lifecycle/errors.go create mode 100644 pkg/controller/component/lifecycle/kbagent.go rename pkg/{kb_agent/httpserver/errors.go => controller/component/lifecycle/lfa_account.go} (65%) rename pkg/{kb_agent/handlers/errors.go => controller/component/lifecycle/lfa_common.go} (82%) rename pkg/{kb_agent/handlers/grpc_handler.go => controller/component/lifecycle/lfa_component.go} (51%) rename pkg/{kb_agent/util/command_test.go => controller/component/lifecycle/lfa_data.go} (58%) create mode 100644 pkg/controller/component/lifecycle/lfa_member.go create mode 100644 pkg/controller/component/lifecycle/lfa_role.go create mode 100644 pkg/controller/component/lifecycle/lifecycle.go delete mode 100644 pkg/kb_agent/cronjobs/checkrole.go delete mode 100644 pkg/kb_agent/cronjobs/checkrole_test.go delete mode 100644 pkg/kb_agent/cronjobs/job.go delete mode 100644 pkg/kb_agent/cronjobs/job_test.go delete mode 100644 pkg/kb_agent/cronjobs/manager.go delete mode 100644 pkg/kb_agent/cronjobs/manager_test.go delete mode 100644 pkg/kb_agent/handlers/action.go delete mode 100644 pkg/kb_agent/handlers/action_test.go delete mode 100644 pkg/kb_agent/handlers/exec_handler.go delete mode 100644 pkg/kb_agent/handlers/exec_handler_test.go delete mode 100644 pkg/kb_agent/handlers/grpc_handler_test.go delete mode 100644 pkg/kb_agent/httpserver/apis.go delete mode 100644 pkg/kb_agent/httpserver/apis_test.go delete mode 100644 pkg/kb_agent/httpserver/config.go delete mode 100644 pkg/kb_agent/httpserver/endpoint.go delete mode 100644 pkg/kb_agent/httpserver/server.go delete mode 100644 pkg/kb_agent/httpserver/server_test.go delete mode 100644 pkg/kb_agent/util/command.go delete mode 100644 pkg/kb_agent/util/event_test.go delete mode 100644 pkg/kb_agent/util/types.go create mode 100644 pkg/kbagent/client/client.go create mode 100644 pkg/kbagent/client/client_mock.go rename pkg/{kb_agent/handlers/interface.go => kbagent/client/generate.go} (71%) create mode 100644 pkg/kbagent/client/httpclient.go create mode 100644 pkg/kbagent/proto/proto.go create mode 100644 pkg/kbagent/server/httpserver.go create mode 100644 pkg/kbagent/server/types.go create mode 100644 pkg/kbagent/service/action.go create mode 100644 pkg/kbagent/service/command.go create mode 100644 pkg/kbagent/service/probe.go create mode 100644 pkg/kbagent/service/service.go create mode 100644 pkg/kbagent/setup.go rename pkg/{kb_agent/util/k8s_client.go => kbagent/util/env.go} (63%) rename pkg/{kb_agent => kbagent}/util/event.go (60%) diff --git a/Makefile b/Makefile index 57ab501e6a3..c23640ec1a9 100644 --- a/Makefile +++ b/Makefile @@ -118,7 +118,7 @@ help: ## Display this help. @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) .PHONY: all -all: manager dataprotection lorry reloader ## Make all cmd binaries. +all: manager dataprotection kbagent ## Make all cmd binaries. ##@ Development @@ -277,6 +277,10 @@ manager: cue-fmt generate manager-go-generate test-go-generate build-checks ## B dataprotection: generate test-go-generate build-checks ## Build dataprotection binary. $(GO) build -ldflags=${LD_FLAGS} -o bin/dataprotection ./cmd/dataprotection/main.go +.PHONY: kbagent +kbagent: generate test-go-generate build-checks + $(GO) build -ldflags=${LD_FLAGS} -o bin/kbagent ./cmd/kbagent/main.go + CERT_ROOT_CA ?= $(WEBHOOK_CERT_DIR)/rootCA.key .PHONY: webhook-cert webhook-cert: $(CERT_ROOT_CA) ## Create root CA certificates for admission webhooks testing. diff --git a/apis/apps/v1alpha1/componentdefinition_types.go b/apis/apps/v1alpha1/componentdefinition_types.go index 177d06c38c7..bcb452f7b95 100644 --- a/apis/apps/v1alpha1/componentdefinition_types.go +++ b/apis/apps/v1alpha1/componentdefinition_types.go @@ -1113,12 +1113,9 @@ type ComponentLifecycleActions struct { // It ensures replicas are correctly labeled with their respective roles. // Without this, services that rely on roleSelectors might improperly direct traffic to wrong replicas. // - // The container executing this action has access to following environment variables: + // The container executing this action has access to following variables: // // - KB_POD_FQDN: The FQDN of the Pod whose role is being assessed. - // - KB_SERVICE_PORT: The port used by the database service. - // - KB_SERVICE_USER: The username with the necessary permissions to interact with the database service. - // - KB_SERVICE_PASSWORD: The corresponding password for KB_SERVICE_USER to authenticate with the database service. // // Expected output of this action: // - On Success: The determined role of the replica, which must align with one of the roles specified @@ -1156,15 +1153,10 @@ type ComponentLifecycleActions struct { // implementation, or automatically by the database kernel or a sidecar utility like Patroni that implements // a consensus algorithm. // - // The container executing this action has access to following environment variables: + // The container executing this action has access to following variables: // - // - KB_SERVICE_PORT: The port used by the database service. - // - KB_SERVICE_USER: The username with the necessary permissions to interact with the database service. - // - KB_SERVICE_PASSWORD: The corresponding password for KB_SERVICE_USER to authenticate with the database service. - // - KB_PRIMARY_POD_FQDN: The FQDN of the primary Pod within the replication group. - // - KB_MEMBER_ADDRESSES: A comma-separated list of Pod addresses for all replicas in the group. - // - KB_NEW_MEMBER_POD_NAME: The pod name of the replica being added to the group. - // - KB_NEW_MEMBER_POD_IP: The IP address of the replica being added to the group. + // - KB_JOIN_MEMBER_POD_FQDN: The pod FQDN of the replica being added to the group. + // - KB_JOIN_MEMBER_POD_NAME: The pod name of the replica being added to the group. // // Expected action output: // - On Failure: An error message detailing the reason for any failure encountered @@ -1177,11 +1169,8 @@ type ComponentLifecycleActions struct { // - bash // - -c // - | - // ADDRESS=$(KB_MEMBER_ADDRESSES%%,*) - // HOST=$(echo $ADDRESS | cut -d ':' -f 1) - // PORT=$(echo $ADDRESS | cut -d ':' -f 2) - // CLIENT="mysql -u $KB_SERVICE_USER -p$KB_SERVICE_PASSWORD -P $PORT -h $HOST -e" - // $CLIENT "ALTER SYSTEM ADD SERVER '$KB_NEW_MEMBER_POD_IP:$KB_SERVICE_PORT' ZONE 'zone1'" + // CLIENT="mysql -u $SERVICE_USER -p$SERVICE_PASSWORD -P $SERVICE_PORT -h $SERVICE_HOST -e" + // $CLIENT "ALTER SYSTEM ADD SERVER '$KB_POD_FQDN:$SERVICE_PORT' ZONE 'zone1'" // ``` // // Note: This field is immutable once it has been set. @@ -1198,15 +1187,10 @@ type ComponentLifecycleActions struct { // The process typically includes updating configurations and informing other group members about the removal. // Data migration is generally not part of this action and should be handled separately if needed. // - // The container executing this action has access to following environment variables: + // The container executing this action has access to following variables: // - // - KB_SERVICE_PORT: The port used by the database service. - // - KB_SERVICE_USER: The username with the necessary permissions to interact with the database service. - // - KB_SERVICE_PASSWORD: The corresponding password for KB_SERVICE_USER to authenticate with the database service. - // - KB_PRIMARY_POD_FQDN: The FQDN of the primary Pod within the replication group. - // - KB_MEMBER_ADDRESSES: A comma-separated list of Pod addresses for all replicas in the group. + // - KB_LEAVE_MEMBER_POD_FQDN: The pod name of the replica being removed from the group. // - KB_LEAVE_MEMBER_POD_NAME: The pod name of the replica being removed from the group. - // - KB_LEAVE_MEMBER_POD_IP: The IP address of the replica being removed from the group. // // Expected action output: // - On Failure: An error message, if applicable, indicating why the action failed. @@ -1218,11 +1202,8 @@ type ComponentLifecycleActions struct { // - bash // - -c // - | - // ADDRESS=$(KB_MEMBER_ADDRESSES%%,*) - // HOST=$(echo $ADDRESS | cut -d ':' -f 1) - // PORT=$(echo $ADDRESS | cut -d ':' -f 2) - // CLIENT="mysql -u $KB_SERVICE_USER -p$KB_SERVICE_PASSWORD -P $PORT -h $HOST -e" - // $CLIENT "ALTER SYSTEM DELETE SERVER '$KB_LEAVE_MEMBER_POD_IP:$KB_SERVICE_PORT' ZONE 'zone1'" + // CLIENT="mysql -u $SERVICE_USER -p$SERVICE_PASSWORD -P $SERVICE_PORT -h $SERVICE_HOST -e" + // $CLIENT "ALTER SYSTEM DELETE SERVER '$KB_POD_FQDN:$SERVICE_PORT' ZONE 'zone1'" // ``` // // Note: This field is immutable once it has been set. @@ -1238,9 +1219,6 @@ type ComponentLifecycleActions struct { // The container executing this action has access to following environment variables: // // - KB_POD_FQDN: The FQDN of the replica pod whose role is being checked. - // - KB_SERVICE_PORT: The port used by the database service. - // - KB_SERVICE_USER: The username with the necessary permissions to interact with the database service. - // - KB_SERVICE_PASSWORD: The corresponding password for KB_SERVICE_USER to authenticate with the database service. // // Expected action output: // - On Failure: An error message, if applicable, indicating why the action failed. @@ -1260,9 +1238,6 @@ type ComponentLifecycleActions struct { // The container executing this action has access to following environment variables: // // - KB_POD_FQDN: The FQDN of the replica pod whose role is being checked. - // - KB_SERVICE_PORT: The port used by the database service. - // - KB_SERVICE_USER: The username with the necessary permissions to interact with the database service. - // - KB_SERVICE_PASSWORD: The corresponding password for KB_SERVICE_USER to authenticate with the database service. // // Expected action output: // - On Failure: An error message, if applicable, indicating why the action failed. diff --git a/cmd/kb_agent/main.go b/cmd/kbagent/main.go similarity index 61% rename from cmd/kb_agent/main.go rename to cmd/kbagent/main.go index db6504794e5..eed36cefd25 100644 --- a/cmd/kb_agent/main.go +++ b/cmd/kbagent/main.go @@ -21,6 +21,7 @@ package main import ( "flag" + "fmt" "os" "os/signal" "strings" @@ -34,14 +35,27 @@ import ( ctrl "sigs.k8s.io/controller-runtime" kzap "sigs.k8s.io/controller-runtime/pkg/log/zap" - "github.com/apecloud/kubeblocks/pkg/kb_agent/cronjobs" - "github.com/apecloud/kubeblocks/pkg/kb_agent/handlers" - "github.com/apecloud/kubeblocks/pkg/kb_agent/httpserver" + kbagent "github.com/apecloud/kubeblocks/pkg/kbagent" + "github.com/apecloud/kubeblocks/pkg/kbagent/server" viper "github.com/apecloud/kubeblocks/pkg/viperx" ) +const ( + defaultPort = 3501 + defaultMaxConcurrency = 8 +) + +var serverConfig server.Config + func init() { viper.AutomaticEnv() + + pflag.StringVar(&serverConfig.Address, "address", "0.0.0.0", "The HTTP Server listen address for kb-agent service.") + pflag.StringVar(&serverConfig.UnixDomainSocket, "unix-socket", "", "The path of the Unix Domain Socket for kb-agent service.") + pflag.IntVar(&serverConfig.Port, "port", defaultPort, "The HTTP Server listen port for kb-agent service.") + pflag.IntVar(&serverConfig.Concurrency, "max-concurrency", defaultMaxConcurrency, + fmt.Sprintf("The maximum number of concurrent connections the Server may serve, use the default value %d if <=0.", defaultMaxConcurrency)) + pflag.BoolVar(&serverConfig.Logging, "api-logging", true, "Enable api logging for kb-agent request.") } func main() { @@ -66,26 +80,20 @@ func main() { if strings.EqualFold("debug", viper.GetString("zap-log-level")) { kopts = append(kopts, kzap.RawZapOpts(zap.AddCaller())) } - ctrl.SetLogger(kzap.New(kopts...)) + logger := kzap.New(kopts...) + ctrl.SetLogger(logger) - // init action handlers - err = handlers.InitHandlers() + // initialize kb-agent + services, err := kbagent.Initialize(logger, os.Environ()) if err != nil { panic(errors.Wrap(err, "init action handlers failed")) } - // start cron jobs - jobManager, err := cronjobs.NewManager() - if err != nil { - panic(errors.Wrap(err, "Cron jobs initialize failed")) - } - jobManager.Start() - // start HTTP Server - httpServer := httpserver.NewServer() - err = httpServer.StartNonBlocking() + server := server.NewHTTPServer(logger, serverConfig, services) + err = server.StartNonBlocking() if err != nil { - panic(errors.Wrap(err, "HTTP server initialize failed")) + panic(errors.Wrap(err, "failed to start HTTP server")) } stop := make(chan os.Signal, 1) diff --git a/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml b/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml index 532dee35efb..c76723b576a 100644 --- a/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml +++ b/config/crd/bases/apps.kubeblocks.io_componentdefinitions.yaml @@ -1451,25 +1451,16 @@ spec: command\nimplementation, or automatically by the database kernel or a sidecar utility like Patroni that implements\na consensus algorithm.\n\n\nThe container executing this action has access - to following environment variables:\n\n\n- KB_SERVICE_PORT: - The port used by the database service.\n- KB_SERVICE_USER: The - username with the necessary permissions to interact with the - database service.\n- KB_SERVICE_PASSWORD: The corresponding - password for KB_SERVICE_USER to authenticate with the database - service.\n- KB_PRIMARY_POD_FQDN: The FQDN of the primary Pod - within the replication group.\n- KB_MEMBER_ADDRESSES: A comma-separated - list of Pod addresses for all replicas in the group.\n- KB_NEW_MEMBER_POD_NAME: - The pod name of the replica being added to the group.\n- KB_NEW_MEMBER_POD_IP: - The IP address of the replica being added to the group.\n\n\nExpected + to following variables:\n\n\n- KB_JOIN_MEMBER_POD_FQDN: The + pod FQDN of the replica being added to the group.\n- KB_JOIN_MEMBER_POD_NAME: + The pod name of the replica being added to the group.\n\n\nExpected action output:\n- On Failure: An error message detailing the reason for any failure encountered\n during the addition of the new member.\n\n\nFor example, to add a new OBServer to an OceanBase Cluster in 'zone1', the following command may be used:\n\n\n```yaml\ncommand:\n- - bash\n- -c\n- |\n ADDRESS=$(KB_MEMBER_ADDRESSES%%,*)\n HOST=$(echo - $ADDRESS | cut -d ':' -f 1)\n PORT=$(echo $ADDRESS | cut -d - ':' -f 2)\n CLIENT=\"mysql -u $KB_SERVICE_USER -p$KB_SERVICE_PASSWORD - -P $PORT -h $HOST -e\"\n\t $CLIENT \"ALTER SYSTEM ADD SERVER - '$KB_NEW_MEMBER_POD_IP:$KB_SERVICE_PORT' ZONE 'zone1'\"\n```\n\n\nNote: + bash\n- -c\n- |\n CLIENT=\"mysql -u $SERVICE_USER -p$SERVICE_PASSWORD + -P $SERVICE_PORT -h $SERVICE_HOST -e\"\n\t $CLIENT \"ALTER + SYSTEM ADD SERVER '$KB_POD_FQDN:$SERVICE_PORT' ZONE 'zone1'\"\n```\n\n\nNote: This field is immutable once it has been set." properties: builtinHandler: @@ -1807,26 +1798,17 @@ spec: includes updating configurations and informing other group members about the removal.\nData migration is generally not part of this action and should be handled separately if needed.\n\n\nThe - container executing this action has access to following environment - variables:\n\n\n- KB_SERVICE_PORT: The port used by the database - service.\n- KB_SERVICE_USER: The username with the necessary - permissions to interact with the database service.\n- KB_SERVICE_PASSWORD: - The corresponding password for KB_SERVICE_USER to authenticate - with the database service.\n- KB_PRIMARY_POD_FQDN: The FQDN - of the primary Pod within the replication group.\n- KB_MEMBER_ADDRESSES: - A comma-separated list of Pod addresses for all replicas in - the group.\n- KB_LEAVE_MEMBER_POD_NAME: The pod name of the - replica being removed from the group.\n- KB_LEAVE_MEMBER_POD_IP: - The IP address of the replica being removed from the group.\n\n\nExpected + container executing this action has access to following variables:\n\n\n- + KB_LEAVE_MEMBER_POD_FQDN: The pod name of the replica being + removed from the group.\n- KB_LEAVE_MEMBER_POD_NAME: The pod + name of the replica being removed from the group.\n\n\nExpected action output:\n- On Failure: An error message, if applicable, indicating why the action failed.\n\n\nFor example, to remove an OBServer from an OceanBase Cluster in 'zone1', the following command can be executed:\n\n\n```yaml\ncommand:\n- bash\n- -c\n- - |\n ADDRESS=$(KB_MEMBER_ADDRESSES%%,*)\n HOST=$(echo $ADDRESS - | cut -d ':' -f 1)\n PORT=$(echo $ADDRESS | cut -d ':' -f - 2)\n CLIENT=\"mysql -u $KB_SERVICE_USER -p$KB_SERVICE_PASSWORD - -P $PORT -h $HOST -e\"\n\t $CLIENT \"ALTER SYSTEM DELETE SERVER - '$KB_LEAVE_MEMBER_POD_IP:$KB_SERVICE_PORT' ZONE 'zone1'\"\n```\n\n\nNote: + |\n CLIENT=\"mysql -u $SERVICE_USER -p$SERVICE_PASSWORD -P + $SERVICE_PORT -h $SERVICE_HOST -e\"\n\t $CLIENT \"ALTER SYSTEM + DELETE SERVER '$KB_POD_FQDN:$SERVICE_PORT' ZONE 'zone1'\"\n```\n\n\nNote: This field is immutable once it has been set." properties: builtinHandler: @@ -2913,9 +2895,6 @@ spec: - KB_POD_FQDN: The FQDN of the replica pod whose role is being checked. - - KB_SERVICE_PORT: The port used by the database service. - - KB_SERVICE_USER: The username with the necessary permissions to interact with the database service. - - KB_SERVICE_PASSWORD: The corresponding password for KB_SERVICE_USER to authenticate with the database service. Expected action output: @@ -3265,9 +3244,6 @@ spec: - KB_POD_FQDN: The FQDN of the replica pod whose role is being checked. - - KB_SERVICE_PORT: The port used by the database service. - - KB_SERVICE_USER: The username with the necessary permissions to interact with the database service. - - KB_SERVICE_PASSWORD: The corresponding password for KB_SERVICE_USER to authenticate with the database service. Expected action output: @@ -3956,13 +3932,10 @@ spec: Without this, services that rely on roleSelectors might improperly direct traffic to wrong replicas. - The container executing this action has access to following environment variables: + The container executing this action has access to following variables: - KB_POD_FQDN: The FQDN of the Pod whose role is being assessed. - - KB_SERVICE_PORT: The port used by the database service. - - KB_SERVICE_USER: The username with the necessary permissions to interact with the database service. - - KB_SERVICE_PASSWORD: The corresponding password for KB_SERVICE_USER to authenticate with the database service. Expected output of this action: diff --git a/controllers/apps/component_controller_test.go b/controllers/apps/component_controller_test.go index 645a23dc6f7..31c74a07a92 100644 --- a/controllers/apps/component_controller_test.go +++ b/controllers/apps/component_controller_test.go @@ -57,7 +57,8 @@ import ( intctrlutil "github.com/apecloud/kubeblocks/pkg/controllerutil" dptypes "github.com/apecloud/kubeblocks/pkg/dataprotection/types" "github.com/apecloud/kubeblocks/pkg/generics" - lorry "github.com/apecloud/kubeblocks/pkg/lorry/client" + kbagent "github.com/apecloud/kubeblocks/pkg/kbagent/client" + kbagentproto "github.com/apecloud/kubeblocks/pkg/kbagent/proto" testapps "github.com/apecloud/kubeblocks/pkg/testutil/apps" testdp "github.com/apecloud/kubeblocks/pkg/testutil/dataprotection" testk8s "github.com/apecloud/kubeblocks/pkg/testutil/k8s" @@ -75,38 +76,36 @@ var ( podAnnotationKey4Test = fmt.Sprintf("%s-test", constant.ComponentReplicasAnnotationKey) ) -var mockLorryClient = func(mock func(*lorry.MockClientMockRecorder)) { - mockLorryCli := lorry.GetMockClient() - if mockLorryCli == nil { - ctrl := gomock.NewController(GinkgoT()) - mockLorryCli = lorry.NewMockClient(ctrl) - } +var mockKBAgentClient = func(mock func(*kbagent.MockClientMockRecorder)) { + cli := kbagent.NewMockClient(gomock.NewController(GinkgoT())) if mock != nil { - mockCli := mockLorryCli.(*lorry.MockClient) - mock(mockCli.EXPECT()) + mock(cli.EXPECT()) } - lorry.SetMockClient(mockLorryCli, nil) + kbagent.SetMockClient(cli, nil) } -var mockLorryClientDefault = func() { - mockLorryClient(func(recorder *lorry.MockClientMockRecorder) { - recorder.CreateUser(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() - recorder.DescribeUser(gomock.Any(), gomock.Any()).Return(nil, nil).AnyTimes() - recorder.GrantUserRole(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() +var mockKBAgentClientDefault = func() { + mockKBAgentClient(func(recorder *kbagent.MockClientMockRecorder) { + recorder.CallAction(gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, req kbagentproto.ActionRequest) (kbagentproto.ActionResponse, error) { + return kbagentproto.ActionResponse{}, nil + }).AnyTimes() }) } -var mockLorryClient4HScale = func(clusterKey types.NamespacedName, compName string, replicas int) { - mockLorryClient(func(recorder *lorry.MockClientMockRecorder) { - recorder.JoinMember(gomock.Any()).Return(nil).AnyTimes() - recorder.LeaveMember(gomock.Any()).DoAndReturn(func(ctx context.Context) error { +var mockKBAgentClient4HScale = func(clusterKey types.NamespacedName, compName string, replicas int) { + mockKBAgentClient(func(recorder *kbagent.MockClientMockRecorder) { + recorder.CallAction(gomock.Any(), gomock.Any()).DoAndReturn(func(ctx context.Context, req kbagentproto.ActionRequest) (kbagentproto.ActionResponse, error) { + rsp := kbagentproto.ActionResponse{} + if req.Action != "memberLeave" { + return rsp, nil + } var podList corev1.PodList labels := client.MatchingLabels{ constant.AppInstanceLabelKey: clusterKey.Name, constant.KBAppComponentLabelKey: compName, } - if err := k8sClient.List(ctx, &podList, labels, client.InNamespace(clusterKey.Namespace)); err != nil { - return err + if err := testCtx.Cli.List(ctx, &podList, labels, client.InNamespace(clusterKey.Namespace)); err != nil { + return rsp, err } for _, pod := range podList.Items { if pod.Annotations == nil { @@ -116,13 +115,12 @@ var mockLorryClient4HScale = func(clusterKey types.NamespacedName, compName stri continue } pod.Annotations[podAnnotationKey4Test] = fmt.Sprintf("%d", replicas) - if err := k8sClient.Update(ctx, &pod); err != nil { - return err + if err := testCtx.Cli.Update(ctx, &pod); err != nil { + return rsp, err } } - return nil + return rsp, nil }).AnyTimes() - recorder.Switchover(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() }) } @@ -214,8 +212,8 @@ var _ = Describe("Component Controller", func() { Create(&testCtx). GetObject() - By("Mock lorry client for the default transformer of system accounts provision") - mockLorryClientDefault() + By("Mock kb-agent client for the default transformer of system accounts provision") + mockKBAgentClientDefault() } waitForCreatingResourceCompletely := func(clusterKey client.ObjectKey, compNames ...string) { @@ -662,6 +660,7 @@ var _ = Describe("Component Controller", func() { if ordinal >= updatedReplicas { continue } + // The annotation was updated by the mocked member leave action. g.Expect(pod.Annotations[podAnnotationKey4Test]).Should(Equal(fmt.Sprintf("%d", updatedReplicas))) } }).Should(Succeed()) @@ -697,7 +696,7 @@ var _ = Describe("Component Controller", func() { } horizontalScale := func(updatedReplicas int, storageClassName string, policyType *string, compDefNames ...string) { - defer lorry.UnsetMockClient() + defer kbagent.UnsetMockClient() cluster := &appsv1alpha1.Cluster{} Expect(k8sClient.Get(testCtx.Ctx, clusterKey, cluster)).Should(Succeed()) @@ -723,7 +722,7 @@ var _ = Describe("Component Controller", func() { } for i, comp := range cluster.Spec.ComponentSpecs { - mockLorryClient4HScale(clusterKey, comp.Name, updatedReplicas) + mockKBAgentClient4HScale(clusterKey, comp.Name, updatedReplicas) By(fmt.Sprintf("H-scale component %s with policy %v", comp.Name, bpt(comp))) horizontalScaleComp(updatedReplicas, &cluster.Spec.ComponentSpecs[i], storageClassName, bpt(comp)) diff --git a/controllers/apps/component_utils.go b/controllers/apps/component_utils.go index d1bf35918b0..67d791f0d6f 100644 --- a/controllers/apps/component_utils.go +++ b/controllers/apps/component_utils.go @@ -26,7 +26,6 @@ import ( "github.com/apecloud/kubeblocks/apis/workloads/v1alpha1" "github.com/apecloud/kubeblocks/pkg/constant" - "github.com/apecloud/kubeblocks/pkg/controllerutil" viper "github.com/apecloud/kubeblocks/pkg/viperx" ) @@ -47,7 +46,6 @@ func delayUpdatePodSpecSystemFields(obj corev1.PodSpec, pobj *corev1.PodSpec) { for i := range pobj.InitContainers { delayUpdateKubeBlocksToolsImage(obj.InitContainers, &pobj.InitContainers[i]) } - updateLorryContainer(obj.Containers, pobj.Containers) } func updateInstanceSetSystemFields(obj v1alpha1.InstanceSetSpec, pobj *v1alpha1.InstanceSetSpec) { @@ -63,22 +61,6 @@ func updatePodSpecSystemFields(obj corev1.PodSpec, pobj *corev1.PodSpec) { for i := range pobj.Containers { updateKubeBlocksToolsImage(&pobj.Containers[i]) } - - updateLorryContainer(obj.Containers, pobj.Containers) -} - -func updateLorryContainer(containers []corev1.Container, pcontainers []corev1.Container) { - srcLorryContainer := controllerutil.GetLorryContainer(containers) - dstLorryContainer := controllerutil.GetLorryContainer(pcontainers) - if srcLorryContainer == nil || dstLorryContainer == nil { - return - } - for i, c := range pcontainers { - if c.Name == dstLorryContainer.Name { - pcontainers[i] = *srcLorryContainer.DeepCopy() - return - } - } } func delayUpdateKubeBlocksToolsImage(containers []corev1.Container, pc *corev1.Container) { diff --git a/controllers/apps/opsrequest_controller_test.go b/controllers/apps/opsrequest_controller_test.go index 3b2d052fce1..82f34c71718 100644 --- a/controllers/apps/opsrequest_controller_test.go +++ b/controllers/apps/opsrequest_controller_test.go @@ -26,7 +26,6 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "k8s.io/utils/pointer" snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1" "golang.org/x/exp/slices" @@ -36,6 +35,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" @@ -181,16 +181,10 @@ var _ = Describe("OpsRequest Controller", func() { By("wait for VerticalScalingOpsRequest is running") Eventually(testapps.GetOpsRequestPhase(&testCtx, opsKey)).Should(Equal(appsv1alpha1.OpsRunningPhase)) + + By("check cluster & component phase as updating") Eventually(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.UpdatingClusterPhase)) Eventually(testapps.GetClusterComponentPhase(&testCtx, clusterKey, mysqlCompName)).Should(Equal(appsv1alpha1.UpdatingClusterCompPhase)) - // TODO(refactor): try to check some ephemeral states? - // checkLatestOpsIsProcessing(clusterKey, verticalScalingOpsRequest.Spec.Type) - - // By("check Cluster and changed component phase is VerticalScaling") - // Eventually(testapps.CheckObj(&testCtx, clusterKey, func(g Gomega, cluster *appsv1alpha1.Cluster) { - // g.Expect(cluster.Status.Phase).To(Equal(appsv1alpha1.SpecReconcilingClusterPhase)) - // g.Expect(cluster.Status.Components[mysqlCompName].Phase).To(Equal(appsv1alpha1.SpecReconcilingClusterCompPhase)) - // })).Should(Succeed()) By("mock bring Cluster and changed component back to running status") Expect(testapps.GetAndChangeObjStatus(&testCtx, client.ObjectKeyFromObject(mysqlIts), func(tmpIts *workloads.InstanceSet) { @@ -198,7 +192,6 @@ var _ = Describe("OpsRequest Controller", func() { })()).ShouldNot(HaveOccurred()) Eventually(testapps.GetClusterComponentPhase(&testCtx, clusterKey, mysqlCompName)).Should(Equal(appsv1alpha1.RunningClusterCompPhase)) Eventually(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.RunningClusterPhase)) - // checkLatestOpsHasProcessed(clusterKey) By("notice opsrequest controller to run") testk8s.MockPodIsTerminating(ctx, testCtx, pod) @@ -261,8 +254,16 @@ var _ = Describe("OpsRequest Controller", func() { Create(&testCtx). GetObject() - By("Mock lorry client for the default transformer of system accounts provision") - mockLorryClientDefault() + By("Create a componentDefinition obj") + compDefObj = testapps.NewComponentDefinitionFactory(compDefName). + WithRandomName(). + AddAnnotations(constant.SkipImmutableCheckAnnotationKey, "true"). + SetDefaultSpec(). + Create(&testCtx). + GetObject() + + By("Mock kb-agent client for the default transformer of system accounts provision") + mockKBAgentClientDefault() }) componentWorkload := func() client.Object { @@ -309,16 +310,17 @@ var _ = Describe("OpsRequest Controller", func() { createMysqlCluster := func(replicas int32) { createBackupPolicyTpl(compDefObj.GetName()) - By("set component to horizontal with snapshot policy and create a cluster") + By("set component to horizontal with snapshot policy") testk8s.MockEnableVolumeSnapshot(&testCtx, testk8s.DefaultStorageClassName) - // TODO(v1.0): bpt - // if clusterDefObj.Spec.ComponentDefs[0].HorizontalScalePolicy == nil { - // Expect(testapps.GetAndChangeObj(&testCtx, client.ObjectKeyFromObject(clusterDefObj), - // func(clusterDef *appsv1alpha1.ClusterDefinition) { - // clusterDef.Spec.ComponentDefs[0].HorizontalScalePolicy = - // &appsv1alpha1.HorizontalScalePolicy{Type: appsv1alpha1.HScaleDataClonePolicyCloneVolume} - // })()).ShouldNot(HaveOccurred()) - // } + + Expect(testapps.GetAndChangeObj(&testCtx, client.ObjectKeyFromObject(compDefObj), + func(compDef *appsv1alpha1.ComponentDefinition) { + if compDef.Annotations == nil { + compDef.Annotations = map[string]string{} + } + compDef.Annotations[constant.HorizontalScaleBackupPolicyTemplateKey] = backupPolicyTPLName + })()).ShouldNot(HaveOccurred()) + pvcSpec := testapps.NewPVCSpec("1Gi") clusterObj = testapps.NewClusterFactory(testCtx.DefaultNamespace, clusterNamePrefix, ""). WithRandomName(). @@ -346,12 +348,17 @@ var _ = Describe("OpsRequest Controller", func() { pvc.Status.Phase = corev1.ClaimBound })()).ShouldNot(HaveOccurred()) } - // wait for cluster observed generation - Eventually(testapps.GetClusterObservedGeneration(&testCtx, clusterKey)).Should(BeEquivalentTo(1)) + + // mock and wait for the cluster running + Eventually(testapps.CheckObj(&testCtx, clusterKey, func(g Gomega, cluster *appsv1alpha1.Cluster) { + g.Expect(cluster.Status.ObservedGeneration).Should(Equal(cluster.Generation)) + })).Should(Succeed()) mockSetClusterStatusPhaseToRunning(clusterKey) - Eventually(testapps.GetClusterComponentPhase(&testCtx, clusterKey, mysqlCompName)).Should(Equal(appsv1alpha1.RunningClusterCompPhase)) - Eventually(testapps.GetClusterPhase(&testCtx, clusterKey)).Should(Equal(appsv1alpha1.RunningClusterPhase)) - Eventually(testapps.GetClusterObservedGeneration(&testCtx, clusterKey)).Should(BeEquivalentTo(1)) + Eventually(testapps.CheckObj(&testCtx, clusterKey, func(g Gomega, cluster *appsv1alpha1.Cluster) { + g.Expect(cluster.Status.ObservedGeneration).Should(Equal(cluster.Generation)) + g.Expect(cluster.Status.Phase).Should(Equal(appsv1alpha1.RunningClusterPhase)) + g.Expect(cluster.Status.Components[mysqlCompName].Phase).Should(Equal(appsv1alpha1.RunningClusterCompPhase)) + })).Should(Succeed()) } createClusterHscaleOps := func(replicas int32) *appsv1alpha1.OpsRequest { @@ -544,8 +551,9 @@ var _ = Describe("OpsRequest Controller", func() { }) It("HorizontalScaling when the number of pods is inconsistent with the number of replicas", func() { - mockLorryClient4HScale(clusterKey, mysqlCompName, 2) + mockKBAgentClient4HScale(clusterKey, mysqlCompName, 2) + // create it with new API since the scale-in operation depends on the member leave action. By("create a cluster with 3 pods") createMysqlCluster(3) diff --git a/controllers/apps/transformer_component_account_provision.go b/controllers/apps/transformer_component_account_provision.go index c38a31585d0..6713f406828 100644 --- a/controllers/apps/transformer_component_account_provision.go +++ b/controllers/apps/transformer_component_account_provision.go @@ -33,11 +33,9 @@ import ( "github.com/apecloud/kubeblocks/pkg/common" "github.com/apecloud/kubeblocks/pkg/constant" "github.com/apecloud/kubeblocks/pkg/controller/component" + "github.com/apecloud/kubeblocks/pkg/controller/component/lifecycle" "github.com/apecloud/kubeblocks/pkg/controller/graph" "github.com/apecloud/kubeblocks/pkg/controller/model" - "github.com/apecloud/kubeblocks/pkg/controllerutil" - lorry "github.com/apecloud/kubeblocks/pkg/lorry/client" - lorryModel "github.com/apecloud/kubeblocks/pkg/lorry/engines/models" ) const ( @@ -79,22 +77,14 @@ func (t *componentAccountProvisionTransformer) Transform(ctx graph.TransformCont return nil } - // TODO: support custom handler for account - // TODO: build lorry client if accountProvision is built-in - lorryCli, err := t.buildLorryClient(transCtx) + lfa, err := t.lifecycleAction(transCtx) if err != nil { return err } - if controllerutil.IsNil(lorryCli) { - return nil - } for _, account := range transCtx.SynthesizeComponent.SystemAccounts { // The secret of initAccount should be rendered into the config file, // or injected into the container through specific account&password environment variables name supported by the engine. // When the engine starts up, it will automatically load and create this account. - // There's no need for lorry to create it again. - // - // InitAccount is necessary because lorry itself requires an account to connect in the first place. if account.InitAccount { continue } @@ -104,7 +94,7 @@ func (t *componentAccountProvisionTransformer) Transform(ctx graph.TransformCont if transCtx.SynthesizeComponent.Annotations[constant.RestoreFromBackupAnnotationKey] == "" { // TODO: restore account secret from backup. // provision account when the component is not recovered from backup - if err = t.provisionAccount(transCtx, cond, lorryCli, account); err != nil { + if err = t.provisionAccount(transCtx, cond, lfa, account); err != nil { t.markProvisionAsFailed(transCtx, &cond, err) return err } @@ -186,9 +176,10 @@ func (t *componentAccountProvisionTransformer) markAccountProvisioned(cond *meta cond.Message = strings.Join(accounts, ",") } -func (t *componentAccountProvisionTransformer) buildLorryClient(transCtx *componentTransformContext) (lorry.Client, error) { +func (t *componentAccountProvisionTransformer) lifecycleAction(transCtx *componentTransformContext) (lifecycle.Lifecycle, error) { synthesizedComp := transCtx.SynthesizeComponent + // TODO(v1.0): remove this, and use the role selector in lifecycle action. roleName := "" for _, role := range synthesizedComp.Roles { if role.Serviceable && role.Writable { @@ -208,15 +199,15 @@ func (t *componentAccountProvisionTransformer) buildLorryClient(transCtx *compon return nil, fmt.Errorf("unable to find appropriate pods to create accounts") } - lorryCli, err := lorry.NewClient(*pods[0]) + lfa, err := lifecycle.New(transCtx.SynthesizeComponent, pods[0]) if err != nil { return nil, err } - return lorryCli, nil + return lfa, nil } func (t *componentAccountProvisionTransformer) provisionAccount(transCtx *componentTransformContext, - _ metav1.Condition, lorryCli lorry.Client, account appsv1alpha1.SystemAccount) error { + _ metav1.Condition, lfa lifecycle.Lifecycle, account appsv1alpha1.SystemAccount) error { synthesizedComp := transCtx.SynthesizeComponent secret, err := t.getAccountSecret(transCtx, synthesizedComp, account) @@ -229,15 +220,13 @@ func (t *componentAccountProvisionTransformer) provisionAccount(transCtx *compon return nil } - userInfo, err := lorryCli.DescribeUser(transCtx, string(username)) - if err == nil && len(userInfo) != 0 { - return nil + vars := map[string]string{ + "$(USERNAME)": string(username), + "$(PASSWD)": string(password), } - - namedVars := getEnvReplacementMapForAccount(string(username), string(password)) - stmt := component.ReplaceNamedVars(namedVars, account.Statement, -1, true) - // TODO: re-define the role - return lorryCli.CreateUser(transCtx.Context, string(username), string(password), string(lorryModel.SuperUserRole), stmt) + stmt := component.ReplaceNamedVars(vars, account.Statement, -1, true) + err = lfa.AccountProvision(transCtx.Context, transCtx.Client, nil, string(username), string(password), stmt) + return lifecycle.IgnoreNotDefined(err) } func (t *componentAccountProvisionTransformer) getAccountSecret(ctx graph.TransformContext, @@ -252,10 +241,3 @@ func (t *componentAccountProvisionTransformer) getAccountSecret(ctx graph.Transf } return secret, nil } - -func getEnvReplacementMapForAccount(name, passwd string) map[string]string { - return map[string]string{ - "$(USERNAME)": name, - "$(PASSWD)": passwd, - } -} diff --git a/controllers/apps/transformer_component_hostnetwork.go b/controllers/apps/transformer_component_hostnetwork.go index ca327cb3c85..8663ead676e 100644 --- a/controllers/apps/transformer_component_hostnetwork.go +++ b/controllers/apps/transformer_component_hostnetwork.go @@ -20,13 +20,8 @@ along with this program. If not, see . package apps import ( - "slices" - "strconv" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/intstr" - "github.com/apecloud/kubeblocks/pkg/constant" "github.com/apecloud/kubeblocks/pkg/controller/component" "github.com/apecloud/kubeblocks/pkg/controller/graph" "github.com/apecloud/kubeblocks/pkg/controller/model" @@ -107,7 +102,6 @@ func allocateHostPortsWithFunc(pm *intctrlutil.PortManager, synthesizedComp *com func updateObjectsWithAllocatedPorts(synthesizedComp *component.SynthesizedComponent, ports map[string]map[string]int32) error { synthesizedComp.PodSpec.HostNetwork = true synthesizedComp.PodSpec.DNSPolicy = corev1.DNSClusterFirstWithHostNet - for i, c := range synthesizedComp.PodSpec.Containers { containerPorts, ok := ports[c.Name] if ok { @@ -118,100 +112,6 @@ func updateObjectsWithAllocatedPorts(synthesizedComp *component.SynthesizedCompo } } } - if err := updateLorrySpecAfterPortsChanged(synthesizedComp); err != nil { - return err - } - return nil -} - -func updateLorrySpecAfterPortsChanged(synthesizeComp *component.SynthesizedComponent) error { - lorryContainer := intctrlutil.GetLorryContainer(synthesizeComp.PodSpec.Containers) - if lorryContainer == nil { - return nil - } - - lorryHTTPPort := getLorryHTTPPort(lorryContainer) - lorryGRPCPort := getLorryGRPCPort(lorryContainer) - if err := updateLorry(synthesizeComp, lorryContainer, lorryHTTPPort, lorryGRPCPort); err != nil { - return err - } - - if err := updateReadinessProbe(synthesizeComp, lorryHTTPPort); err != nil { - return err - } + component.UpdateKBAgentContainer4HostNetwork(synthesizedComp) return nil } - -func updateLorry(synthesizeComp *component.SynthesizedComponent, container *corev1.Container, httpPort, grpcPort int) error { - kbLorryBinary := "/kubeblocks/lorry" - if slices.Contains(container.Command, kbLorryBinary) { - container.Command = []string{kbLorryBinary, - "--port", strconv.Itoa(httpPort), - "--grpcport", strconv.Itoa(grpcPort), - "--config-path", "/kubeblocks/config/lorry/components/", - } - } else { - container.Command = []string{"lorry", - "--port", strconv.Itoa(httpPort), - "--grpcport", strconv.Itoa(grpcPort), - } - } - if container.StartupProbe != nil && container.StartupProbe.TCPSocket != nil { - container.StartupProbe.TCPSocket.Port = intstr.FromInt(httpPort) - } - - for i := range container.Env { - if container.Env[i].Name != constant.KBEnvServicePort { - continue - } - if len(synthesizeComp.PodSpec.Containers) > 0 { - mainContainer := synthesizeComp.PodSpec.Containers[0] - if len(mainContainer.Ports) > 0 { - port := mainContainer.Ports[0] - dbPort := port.ContainerPort - container.Env[i] = corev1.EnvVar{ - Name: constant.KBEnvServicePort, - Value: strconv.Itoa(int(dbPort)), - ValueFrom: nil, - } - } - } - } - return nil -} - -func updateReadinessProbe(synthesizeComp *component.SynthesizedComponent, lorryHTTPPort int) error { - var container *corev1.Container - for i := range synthesizeComp.PodSpec.Containers { - container = &synthesizeComp.PodSpec.Containers[i] - if container.ReadinessProbe == nil { - continue - } - if container.ReadinessProbe.HTTPGet == nil { - continue - } - if container.ReadinessProbe.HTTPGet.Path == constant.LorryRoleProbePath || - container.ReadinessProbe.HTTPGet.Path == constant.LorryVolumeProtectPath { - container.ReadinessProbe.HTTPGet.Port = intstr.FromInt(lorryHTTPPort) - } - } - return nil -} - -func getLorryHTTPPort(container *corev1.Container) int { - for _, port := range container.Ports { - if port.Name == constant.LorryHTTPPortName { - return int(port.ContainerPort) - } - } - return 0 -} - -func getLorryGRPCPort(container *corev1.Container) int { - for _, port := range container.Ports { - if port.Name == constant.LorryGRPCPortName { - return int(port.ContainerPort) - } - } - return 0 -} diff --git a/controllers/apps/transformer_component_rbac.go b/controllers/apps/transformer_component_rbac.go index 50d10b36db9..1a61dbe9d8b 100644 --- a/controllers/apps/transformer_component_rbac.go +++ b/controllers/apps/transformer_component_rbac.go @@ -105,10 +105,6 @@ func (t *componentRBACTransformer) Transform(ctx graph.TransformContext, dag *gr } func isLifecycleActionsEnabled(compDef *appsv1alpha1.ComponentDefinition) bool { - // TODO(component): in KB 0.9, LifeCycleActions is executed throuth lorry, and - // lorry requires the service account to have the permission defined in "kubeblocks-cluster-pod-role". - // In KB 1.0, LifeCycleActions is executed throuth the kb-agent, and it does not require the permission anymore. - // So, we can remove this check in KB 1.0. return compDef.Spec.LifecycleActions != nil } diff --git a/controllers/apps/transformer_component_workload.go b/controllers/apps/transformer_component_workload.go index 95637db4f9e..a039d4d6228 100644 --- a/controllers/apps/transformer_component_workload.go +++ b/controllers/apps/transformer_component_workload.go @@ -21,6 +21,7 @@ package apps import ( "context" + "errors" "fmt" "reflect" "strings" @@ -39,12 +40,12 @@ import ( workloads "github.com/apecloud/kubeblocks/apis/workloads/v1alpha1" "github.com/apecloud/kubeblocks/pkg/constant" "github.com/apecloud/kubeblocks/pkg/controller/component" + "github.com/apecloud/kubeblocks/pkg/controller/component/lifecycle" "github.com/apecloud/kubeblocks/pkg/controller/configuration" "github.com/apecloud/kubeblocks/pkg/controller/factory" "github.com/apecloud/kubeblocks/pkg/controller/graph" "github.com/apecloud/kubeblocks/pkg/controller/model" intctrlutil "github.com/apecloud/kubeblocks/pkg/controllerutil" - lorry "github.com/apecloud/kubeblocks/pkg/lorry/client" ) // componentWorkloadTransformer handles component workload generation @@ -387,8 +388,8 @@ func checkNRollbackProtoImages(itsObj, itsProto *workloads.InstanceSet) { for i, cc := range [][]corev1.Container{itsObj.Spec.Template.Spec.InitContainers, itsObj.Spec.Template.Spec.Containers} { images[i] = make(map[string]string) for _, c := range cc { - // skip the lorry container - if c.Name == constant.LorryInitContainerName || c.Name == constant.LorryContainerName { + // skip the kb-agent container + if component.IsKBAgentContainer(&c) { continue } images[i][c.Name] = c.Image @@ -600,26 +601,19 @@ func (r *componentWorkloadOps) leaveMember4ScaleIn() error { return false } - tryToSwitchover := func(lorryCli lorry.Client, pod *corev1.Pod) error { + tryToSwitchover := func(lfa lifecycle.Lifecycle, pod *corev1.Pod) error { // if pod is not leader/primary, no need to switchover if !isLeader(pod) { return nil } // if HA functionality is not enabled, no need to switchover - err := lorryCli.Switchover(r.reqCtx.Ctx, pod.Name, "", false) - if err == lorry.NotImplemented { - // For the purpose of upgrade compatibility, if the version of Lorry is 0.7 and - // the version of KB is upgraded to 0.8 or newer, lorry client will return an NotImplemented error, - // in this case, here just return success. - r.reqCtx.Log.Info("lorry switchover api is not implemented") + err := lfa.Switchover(r.reqCtx.Ctx, r.cli, nil) + if err != nil && errors.Is(err, lifecycle.ErrActionNotDefined) { return nil } if err == nil { return fmt.Errorf("switchover succeed, wait role label to be updated") } - if strings.Contains(err.Error(), "cluster's ha is disabled") { - return nil - } return err } @@ -638,7 +632,7 @@ func (r *componentWorkloadOps) leaveMember4ScaleIn() error { continue } - lorryCli, err1 := lorry.NewClient(*pod) + lfa, err1 := lifecycle.New(r.synthesizeComp, pod, pods...) if err1 != nil { if err == nil { err = err1 @@ -646,23 +640,13 @@ func (r *componentWorkloadOps) leaveMember4ScaleIn() error { continue } - if intctrlutil.IsNil(lorryCli) { - // no lorry in the pod - continue - } - // switchover if the leaving pod is leader - if switchoverErr := tryToSwitchover(lorryCli, pod); switchoverErr != nil { + if switchoverErr := tryToSwitchover(lfa, pod); switchoverErr != nil { return switchoverErr } - if err2 := lorryCli.LeaveMember(r.reqCtx.Ctx); err2 != nil { - // For the purpose of upgrade compatibility, if the version of Lorry is 0.7 and - // the version of KB is upgraded to 0.8 or newer, lorry client will return an NotImplemented error, - // in this case, here just ignore it. - if err2 == lorry.NotImplemented { - r.reqCtx.Log.Info("lorry leave member api is not implemented") - } else if err == nil { + if err2 := lfa.MemberLeave(r.reqCtx.Ctx, r.cli, nil); err2 != nil { + if !errors.Is(err2, lifecycle.ErrActionNotDefined) && err == nil { err = err2 } } diff --git a/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml b/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml index 532dee35efb..c76723b576a 100644 --- a/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml +++ b/deploy/helm/crds/apps.kubeblocks.io_componentdefinitions.yaml @@ -1451,25 +1451,16 @@ spec: command\nimplementation, or automatically by the database kernel or a sidecar utility like Patroni that implements\na consensus algorithm.\n\n\nThe container executing this action has access - to following environment variables:\n\n\n- KB_SERVICE_PORT: - The port used by the database service.\n- KB_SERVICE_USER: The - username with the necessary permissions to interact with the - database service.\n- KB_SERVICE_PASSWORD: The corresponding - password for KB_SERVICE_USER to authenticate with the database - service.\n- KB_PRIMARY_POD_FQDN: The FQDN of the primary Pod - within the replication group.\n- KB_MEMBER_ADDRESSES: A comma-separated - list of Pod addresses for all replicas in the group.\n- KB_NEW_MEMBER_POD_NAME: - The pod name of the replica being added to the group.\n- KB_NEW_MEMBER_POD_IP: - The IP address of the replica being added to the group.\n\n\nExpected + to following variables:\n\n\n- KB_JOIN_MEMBER_POD_FQDN: The + pod FQDN of the replica being added to the group.\n- KB_JOIN_MEMBER_POD_NAME: + The pod name of the replica being added to the group.\n\n\nExpected action output:\n- On Failure: An error message detailing the reason for any failure encountered\n during the addition of the new member.\n\n\nFor example, to add a new OBServer to an OceanBase Cluster in 'zone1', the following command may be used:\n\n\n```yaml\ncommand:\n- - bash\n- -c\n- |\n ADDRESS=$(KB_MEMBER_ADDRESSES%%,*)\n HOST=$(echo - $ADDRESS | cut -d ':' -f 1)\n PORT=$(echo $ADDRESS | cut -d - ':' -f 2)\n CLIENT=\"mysql -u $KB_SERVICE_USER -p$KB_SERVICE_PASSWORD - -P $PORT -h $HOST -e\"\n\t $CLIENT \"ALTER SYSTEM ADD SERVER - '$KB_NEW_MEMBER_POD_IP:$KB_SERVICE_PORT' ZONE 'zone1'\"\n```\n\n\nNote: + bash\n- -c\n- |\n CLIENT=\"mysql -u $SERVICE_USER -p$SERVICE_PASSWORD + -P $SERVICE_PORT -h $SERVICE_HOST -e\"\n\t $CLIENT \"ALTER + SYSTEM ADD SERVER '$KB_POD_FQDN:$SERVICE_PORT' ZONE 'zone1'\"\n```\n\n\nNote: This field is immutable once it has been set." properties: builtinHandler: @@ -1807,26 +1798,17 @@ spec: includes updating configurations and informing other group members about the removal.\nData migration is generally not part of this action and should be handled separately if needed.\n\n\nThe - container executing this action has access to following environment - variables:\n\n\n- KB_SERVICE_PORT: The port used by the database - service.\n- KB_SERVICE_USER: The username with the necessary - permissions to interact with the database service.\n- KB_SERVICE_PASSWORD: - The corresponding password for KB_SERVICE_USER to authenticate - with the database service.\n- KB_PRIMARY_POD_FQDN: The FQDN - of the primary Pod within the replication group.\n- KB_MEMBER_ADDRESSES: - A comma-separated list of Pod addresses for all replicas in - the group.\n- KB_LEAVE_MEMBER_POD_NAME: The pod name of the - replica being removed from the group.\n- KB_LEAVE_MEMBER_POD_IP: - The IP address of the replica being removed from the group.\n\n\nExpected + container executing this action has access to following variables:\n\n\n- + KB_LEAVE_MEMBER_POD_FQDN: The pod name of the replica being + removed from the group.\n- KB_LEAVE_MEMBER_POD_NAME: The pod + name of the replica being removed from the group.\n\n\nExpected action output:\n- On Failure: An error message, if applicable, indicating why the action failed.\n\n\nFor example, to remove an OBServer from an OceanBase Cluster in 'zone1', the following command can be executed:\n\n\n```yaml\ncommand:\n- bash\n- -c\n- - |\n ADDRESS=$(KB_MEMBER_ADDRESSES%%,*)\n HOST=$(echo $ADDRESS - | cut -d ':' -f 1)\n PORT=$(echo $ADDRESS | cut -d ':' -f - 2)\n CLIENT=\"mysql -u $KB_SERVICE_USER -p$KB_SERVICE_PASSWORD - -P $PORT -h $HOST -e\"\n\t $CLIENT \"ALTER SYSTEM DELETE SERVER - '$KB_LEAVE_MEMBER_POD_IP:$KB_SERVICE_PORT' ZONE 'zone1'\"\n```\n\n\nNote: + |\n CLIENT=\"mysql -u $SERVICE_USER -p$SERVICE_PASSWORD -P + $SERVICE_PORT -h $SERVICE_HOST -e\"\n\t $CLIENT \"ALTER SYSTEM + DELETE SERVER '$KB_POD_FQDN:$SERVICE_PORT' ZONE 'zone1'\"\n```\n\n\nNote: This field is immutable once it has been set." properties: builtinHandler: @@ -2913,9 +2895,6 @@ spec: - KB_POD_FQDN: The FQDN of the replica pod whose role is being checked. - - KB_SERVICE_PORT: The port used by the database service. - - KB_SERVICE_USER: The username with the necessary permissions to interact with the database service. - - KB_SERVICE_PASSWORD: The corresponding password for KB_SERVICE_USER to authenticate with the database service. Expected action output: @@ -3265,9 +3244,6 @@ spec: - KB_POD_FQDN: The FQDN of the replica pod whose role is being checked. - - KB_SERVICE_PORT: The port used by the database service. - - KB_SERVICE_USER: The username with the necessary permissions to interact with the database service. - - KB_SERVICE_PASSWORD: The corresponding password for KB_SERVICE_USER to authenticate with the database service. Expected action output: @@ -3956,13 +3932,10 @@ spec: Without this, services that rely on roleSelectors might improperly direct traffic to wrong replicas. - The container executing this action has access to following environment variables: + The container executing this action has access to following variables: - KB_POD_FQDN: The FQDN of the Pod whose role is being assessed. - - KB_SERVICE_PORT: The port used by the database service. - - KB_SERVICE_USER: The username with the necessary permissions to interact with the database service. - - KB_SERVICE_PASSWORD: The corresponding password for KB_SERVICE_USER to authenticate with the database service. Expected output of this action: diff --git a/docker/Dockerfile-tools b/docker/Dockerfile-tools index df02c0de247..071e8186dbd 100644 --- a/docker/Dockerfile-tools +++ b/docker/Dockerfile-tools @@ -71,16 +71,20 @@ RUN --mount=type=bind,target=. \ --mount=type=cache,target=/go/pkg/mod \ CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -ldflags="${LD_FLAGS}" -a -o /out/config_render cmd/reloader/template/*.go -RUN --mount=type=bind,target=. \ - --mount=type=cache,target=/root/.cache/go-build \ - --mount=type=cache,target=/go/pkg/mod \ - CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -ldflags="${LD_FLAGS}" -a -o /out/lorry cmd/lorry/main.go +# RUN --mount=type=bind,target=. \ +# --mount=type=cache,target=/root/.cache/go-build \ +# --mount=type=cache,target=/go/pkg/mod \ +# CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -ldflags="${LD_FLAGS}" -a -o /out/lorry cmd/lorry/main.go + +# RUN --mount=type=bind,target=. \ +# --mount=type=cache,target=/root/.cache/go-build \ +# --mount=type=cache,target=/go/pkg/mod \ +# CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -ldflags="${LD_FLAGS}" -a -o /out/lorryctl cmd/lorry/ctl/main.go RUN --mount=type=bind,target=. \ --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ - CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -ldflags="${LD_FLAGS}" -a -o /out/lorryctl cmd/lorry/ctl/main.go - + CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -ldflags="${LD_FLAGS}" -a -o /out/kbagent cmd/kbagent/main.go RUN GRPC_HEALTH_PROBE_VERSION=v0.4.13 GOOS=${TARGETOS} GOARCH=${TARGETARCH} && \ wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-${GOOS}-${GOARCH} @@ -100,20 +104,21 @@ RUN apk add --no-cache kubectl helm jq --allow-untrusted \ && rm -rf /var/cache/apk/* # copy kubeblocks tools -COPY config/lorry config/lorry +# COPY config/lorry config/lorry COPY config/crd/bases /kubeblocks/crd COPY --from=builder /out/killer /bin COPY --from=builder /out/reloader /bin COPY --from=builder /out/config_render /bin -COPY --from=builder /out/lorry /bin -COPY --from=builder /out/lorryctl /bin +# COPY --from=builder /out/lorry /bin +# COPY --from=builder /out/lorryctl /bin +COPY --from=builder /out/kbagent /bin COPY --from=builder /bin/grpc_health_probe /bin COPY --from=builder /out/helm_hook /bin COPY --from=binary-downloader /bin/curl /bin/ # make breaking change compatible -RUN ln -s /bin/lorry /bin/probe -RUN ln -s /config/lorry /config/probe +# RUN ln -s /bin/lorry /bin/probe +# RUN ln -s /config/lorry /config/probe # enable grpc_health_probe binary RUN chmod +x /bin/grpc_health_probe diff --git a/docs/developer_docs/api-reference/cluster.md b/docs/developer_docs/api-reference/cluster.md index a71d19eaec4..c573cd80d31 100644 --- a/docs/developer_docs/api-reference/cluster.md +++ b/docs/developer_docs/api-reference/cluster.md @@ -6911,12 +6911,9 @@ which initiates an update of the replica’s role.

Defining a RoleProbe Action for a Component is required if roles are defined for the Component. It ensures replicas are correctly labeled with their respective roles. Without this, services that rely on roleSelectors might improperly direct traffic to wrong replicas.

-

The container executing this action has access to following environment variables:

+

The container executing this action has access to following variables:

  • KB_POD_FQDN: The FQDN of the Pod whose role is being assessed.
    • -
  • KB_SERVICE_PORT: The port used by the database service.
    • -
  • KB_SERVICE_USER: The username with the necessary permissions to interact with the database service.
    • -
  • KB_SERVICE_PASSWORD: The corresponding password for KB_SERVICE_USER to authenticate with the database service.

Expected output of this action: - On Success: The determined role of the replica, which must align with one of the roles specified @@ -6967,15 +6964,10 @@ LifecycleActionHandler

The role of the replica (e.g., primary, secondary) will be determined and assigned as part of the action command implementation, or automatically by the database kernel or a sidecar utility like Patroni that implements a consensus algorithm.

-

The container executing this action has access to following environment variables:

+

The container executing this action has access to following variables:

    -
  • KB_SERVICE_PORT: The port used by the database service.
    • -
  • KB_SERVICE_USER: The username with the necessary permissions to interact with the database service.
    • -
  • KB_SERVICE_PASSWORD: The corresponding password for KB_SERVICE_USER to authenticate with the database service.
    • -
  • KB_PRIMARY_POD_FQDN: The FQDN of the primary Pod within the replication group.
    • -
  • KB_MEMBER_ADDRESSES: A comma-separated list of Pod addresses for all replicas in the group.
    • -
  • KB_NEW_MEMBER_POD_NAME: The pod name of the replica being added to the group.
    • -
  • KB_NEW_MEMBER_POD_IP: The IP address of the replica being added to the group.
    • +
  • KB_JOIN_MEMBER_POD_FQDN: The pod FQDN of the replica being added to the group.
    • +
  • KB_JOIN_MEMBER_POD_NAME: The pod name of the replica being added to the group.

Expected action output: - On Failure: An error message detailing the reason for any failure encountered @@ -6985,11 +6977,8 @@ during the addition of the new member.

- bash - -c - | - ADDRESS=$(KB_MEMBER_ADDRESSES%%,*) - HOST=$(echo $ADDRESS | cut -d ':' -f 1) - PORT=$(echo $ADDRESS | cut -d ':' -f 2) - CLIENT="mysql -u $KB_SERVICE_USER -p$KB_SERVICE_PASSWORD -P $PORT -h $HOST -e" - $CLIENT "ALTER SYSTEM ADD SERVER '$KB_NEW_MEMBER_POD_IP:$KB_SERVICE_PORT' ZONE 'zone1'" + CLIENT="mysql -u $SERVICE_USER -p$SERVICE_PASSWORD -P $SERVICE_PORT -h $SERVICE_HOST -e" + $CLIENT "ALTER SYSTEM ADD SERVER '$KB_POD_FQDN:$SERVICE_PORT' ZONE 'zone1'"

Note: This field is immutable once it has been set.

@@ -7011,15 +7000,10 @@ The operator will wait for MemberLeave to complete successfully before releasing related Kubernetes resources.

The process typically includes updating configurations and informing other group members about the removal. Data migration is generally not part of this action and should be handled separately if needed.

-

The container executing this action has access to following environment variables:

+

The container executing this action has access to following variables:

    -
  • KB_SERVICE_PORT: The port used by the database service.
    • -
  • KB_SERVICE_USER: The username with the necessary permissions to interact with the database service.
    • -
  • KB_SERVICE_PASSWORD: The corresponding password for KB_SERVICE_USER to authenticate with the database service.
    • -
  • KB_PRIMARY_POD_FQDN: The FQDN of the primary Pod within the replication group.
    • -
  • KB_MEMBER_ADDRESSES: A comma-separated list of Pod addresses for all replicas in the group.
    • +
  • KB_LEAVE_MEMBER_POD_FQDN: The pod name of the replica being removed from the group.
  • KB_LEAVE_MEMBER_POD_NAME: The pod name of the replica being removed from the group.
    • -
  • KB_LEAVE_MEMBER_POD_IP: The IP address of the replica being removed from the group.

Expected action output: - On Failure: An error message, if applicable, indicating why the action failed.

@@ -7028,11 +7012,8 @@ Data migration is generally not part of this action and should be handled separa - bash - -c - | - ADDRESS=$(KB_MEMBER_ADDRESSES%%,*) - HOST=$(echo $ADDRESS | cut -d ':' -f 1) - PORT=$(echo $ADDRESS | cut -d ':' -f 2) - CLIENT="mysql -u $KB_SERVICE_USER -p$KB_SERVICE_PASSWORD -P $PORT -h $HOST -e" - $CLIENT "ALTER SYSTEM DELETE SERVER '$KB_LEAVE_MEMBER_POD_IP:$KB_SERVICE_PORT' ZONE 'zone1'" + CLIENT="mysql -u $SERVICE_USER -p$SERVICE_PASSWORD -P $SERVICE_PORT -h $SERVICE_HOST -e" + $CLIENT "ALTER SYSTEM DELETE SERVER '$KB_POD_FQDN:$SERVICE_PORT' ZONE 'zone1'"

Note: This field is immutable once it has been set.

@@ -7054,9 +7035,6 @@ This action is invoked when the database’s volume capacity nears its upper

The container executing this action has access to following environment variables:

  • KB_POD_FQDN: The FQDN of the replica pod whose role is being checked.
    • -
  • KB_SERVICE_PORT: The port used by the database service.
    • -
  • KB_SERVICE_USER: The username with the necessary permissions to interact with the database service.
    • -
  • KB_SERVICE_PASSWORD: The corresponding password for KB_SERVICE_USER to authenticate with the database service.

Expected action output: - On Failure: An error message, if applicable, indicating why the action failed.

@@ -7082,9 +7060,6 @@ both read and write operations.

The container executing this action has access to following environment variables:

  • KB_POD_FQDN: The FQDN of the replica pod whose role is being checked.
    • -
  • KB_SERVICE_PORT: The port used by the database service.
    • -
  • KB_SERVICE_USER: The username with the necessary permissions to interact with the database service.
    • -
  • KB_SERVICE_PASSWORD: The corresponding password for KB_SERVICE_USER to authenticate with the database service.

Expected action output: - On Failure: An error message, if applicable, indicating why the action failed.

diff --git a/pkg/controller/component/its_convertor.go b/pkg/controller/component/its_convertor.go index 969e80f1d53..2363f902f1f 100644 --- a/pkg/controller/component/its_convertor.go +++ b/pkg/controller/component/its_convertor.go @@ -278,40 +278,7 @@ func (c *itsRolesConvertor) convert(args ...any) (any, error) { // itsRoleProbeConvertor converts the ComponentDefinition.Spec.LifecycleActions.RoleProbe into InstanceSet.Spec.RoleProbe. func (c *itsRoleProbeConvertor) convert(args ...any) (any, error) { - synthesizeComp, err := parseITSConvertorArgs(args...) - if err != nil { - return nil, err - } - - if synthesizeComp.LifecycleActions == nil || synthesizeComp.LifecycleActions.RoleProbe == nil { - return nil, nil - } - - itsRoleProbe := &workloads.RoleProbe{ - TimeoutSeconds: synthesizeComp.LifecycleActions.RoleProbe.TimeoutSeconds, - PeriodSeconds: synthesizeComp.LifecycleActions.RoleProbe.PeriodSeconds, - SuccessThreshold: 1, - FailureThreshold: 2, - RoleUpdateMechanism: workloads.DirectAPIServerEventUpdate, - } - - if synthesizeComp.LifecycleActions.RoleProbe.BuiltinHandler != nil { - builtinHandler := string(*synthesizeComp.LifecycleActions.RoleProbe.BuiltinHandler) - itsRoleProbe.BuiltinHandler = &builtinHandler - return itsRoleProbe, nil - } - - // TODO(xingran): ITS Action does not support args[] yet - if synthesizeComp.LifecycleActions.RoleProbe.Exec != nil { - itsRoleProbeCmdAction := workloads.Action{ - Image: synthesizeComp.LifecycleActions.RoleProbe.Exec.Image, - Command: synthesizeComp.LifecycleActions.RoleProbe.Exec.Command, - Args: synthesizeComp.LifecycleActions.RoleProbe.Exec.Args, - } - itsRoleProbe.CustomHandler = []workloads.Action{itsRoleProbeCmdAction} - } - - return itsRoleProbe, nil + return nil, nil } func (c *itsCredentialConvertor) convert(args ...any) (any, error) { diff --git a/pkg/controller/component/its_convertor_test.go b/pkg/controller/component/its_convertor_test.go deleted file mode 100644 index f3db70fae51..00000000000 --- a/pkg/controller/component/its_convertor_test.go +++ /dev/null @@ -1,61 +0,0 @@ -/* -Copyright (C) 2022-2024 ApeCloud Co., Ltd - -This file is part of KubeBlocks project - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ - -package component - -import ( - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" - workloadsalpha1 "github.com/apecloud/kubeblocks/apis/workloads/v1alpha1" -) - -var _ = Describe("Test InstanceSet Convertor", func() { - Context("InstanceSet convertors", func() { - var ( - synComp *SynthesizedComponent - ) - command := []string{"foo", "bar"} - args := []string{"zoo", "boo"} - - BeforeEach(func() { - synComp = &SynthesizedComponent{ - LifecycleActions: &appsv1alpha1.ComponentLifecycleActions{ - RoleProbe: &appsv1alpha1.Probe{ - Action: appsv1alpha1.Action{ - Exec: &appsv1alpha1.ExecAction{ - Command: command, - Args: args, - }, - }, - }, - }, - } - }) - It("convert", func() { - convertor := &itsRoleProbeConvertor{} - res, err := convertor.convert(synComp) - Expect(err).Should(Succeed()) - probe := res.(*workloadsalpha1.RoleProbe) - Expect(probe.CustomHandler[0].Command).Should(BeEquivalentTo(command)) - Expect(probe.CustomHandler[0].Args).Should(BeEquivalentTo(args)) - }) - }) -}) diff --git a/pkg/controller/component/kbagent.go b/pkg/controller/component/kbagent.go new file mode 100644 index 00000000000..4925a46ab3c --- /dev/null +++ b/pkg/controller/component/kbagent.go @@ -0,0 +1,389 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package component + +import ( + "errors" + "fmt" + "strconv" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" + + appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + "github.com/apecloud/kubeblocks/pkg/constant" + "github.com/apecloud/kubeblocks/pkg/controller/builder" + intctrlutil "github.com/apecloud/kubeblocks/pkg/controllerutil" + kbagent "github.com/apecloud/kubeblocks/pkg/kbagent" + "github.com/apecloud/kubeblocks/pkg/kbagent/proto" + viper "github.com/apecloud/kubeblocks/pkg/viperx" +) + +const ( + kbAgentContainerName = "kbagent" + kbAgentInitContainerName = "init-kbagent" + kbAgentCommand = "/bin/kbagent" + kbAgentPortName = "http" + + kbAgentSharedMountPath = "/kubeblocks" + kbAgentCommandOnSharedMount = "/kubeblocks/kbagent" + + minAvailablePort = 1025 + maxAvailablePort = 65535 + kbAgentDefaultPort = 3501 +) + +var ( + sharedVolumeMount = corev1.VolumeMount{Name: "kubeblocks", MountPath: kbAgentSharedMountPath} +) + +func IsKBAgentContainer(c *corev1.Container) bool { + return c.Name == kbAgentContainerName || c.Name == kbAgentInitContainerName +} + +func UpdateKBAgentContainer4HostNetwork(synthesizedComp *SynthesizedComponent) { + idx, c := intctrlutil.GetContainerByName(synthesizedComp.PodSpec.Containers, kbAgentContainerName) + if c == nil { + return + } + + httpPort := 0 + for _, port := range c.Ports { + if port.Name == kbAgentPortName { + httpPort = int(port.ContainerPort) + break + } + } + if httpPort == 0 { + return + } + + // update port in args + for i, arg := range c.Args { + if arg == "--port" { + c.Args[i+1] = strconv.Itoa(httpPort) + break + } + } + + // update startup probe + if c.StartupProbe != nil && c.StartupProbe.TCPSocket != nil { + c.StartupProbe.TCPSocket.Port = intstr.FromInt(httpPort) + } + + synthesizedComp.PodSpec.Containers[idx] = *c +} + +func buildKBAgentContainer(synthesizedComp *SynthesizedComponent) error { + if synthesizedComp.LifecycleActions == nil { + return nil + } + + envVars, err := buildKBAgentStartupEnvs(synthesizedComp) + if err != nil { + return err + } + + ports, err := getAvailablePorts(synthesizedComp.PodSpec.Containers, []int32{int32(kbAgentDefaultPort)}) + if err != nil { + return err + } + + port := int(ports[0]) + container := builder.NewContainerBuilder(kbAgentContainerName). + SetImage(viper.GetString(constant.KBToolsImage)). + SetImagePullPolicy(corev1.PullIfNotPresent). + AddCommands(kbAgentCommand). + AddArgs("--port", strconv.Itoa(port)). + AddEnv(envVars...). + AddPorts(corev1.ContainerPort{ + ContainerPort: int32(port), + Name: kbAgentPortName, + Protocol: "TCP", + }). + SetStartupProbe(corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{Port: intstr.FromInt(port)}, + }}). + GetObject() + + if err = adaptKBAgentIfCustomImageNContainerDefined(synthesizedComp, container); err != nil { + return err + } + + // set kb-agent container ports to host network + if synthesizedComp.HostNetwork != nil { + if synthesizedComp.HostNetwork.ContainerPorts == nil { + synthesizedComp.HostNetwork.ContainerPorts = make([]appsv1alpha1.HostNetworkContainerPort, 0) + } + synthesizedComp.HostNetwork.ContainerPorts = append( + synthesizedComp.HostNetwork.ContainerPorts, + appsv1alpha1.HostNetworkContainerPort{ + Container: container.Name, + Ports: []string{kbAgentPortName}, + }) + } + + synthesizedComp.PodSpec.Containers = append(synthesizedComp.PodSpec.Containers, *container) + return nil +} + +func buildKBAgentStartupEnvs(synthesizedComp *SynthesizedComponent) ([]corev1.EnvVar, error) { + var ( + actions []proto.Action + probes []proto.Probe + ) + + if a := buildAction4KBAgent(synthesizedComp.LifecycleActions.PostProvision, "postProvision"); a != nil { + actions = append(actions, *a) + } + if a := buildAction4KBAgent(synthesizedComp.LifecycleActions.PreTerminate, "preTerminate"); a != nil { + actions = append(actions, *a) + } + if synthesizedComp.LifecycleActions.Switchover != nil { + if a := buildAction4KBAgentLow(synthesizedComp.LifecycleActions.Switchover, "switchover"); a != nil { + actions = append(actions, *a) + } + } + if a := buildAction4KBAgent(synthesizedComp.LifecycleActions.MemberJoin, "memberJoin"); a != nil { + actions = append(actions, *a) + } + if a := buildAction4KBAgent(synthesizedComp.LifecycleActions.MemberLeave, "memberLeave"); a != nil { + actions = append(actions, *a) + } + if a := buildAction4KBAgent(synthesizedComp.LifecycleActions.Readonly, "readonly"); a != nil { + actions = append(actions, *a) + } + if a := buildAction4KBAgent(synthesizedComp.LifecycleActions.Readwrite, "readwrite"); a != nil { + actions = append(actions, *a) + } + if a := buildAction4KBAgent(synthesizedComp.LifecycleActions.DataDump, "dataDump"); a != nil { + actions = append(actions, *a) + } + if a := buildAction4KBAgent(synthesizedComp.LifecycleActions.DataLoad, "dataLoad"); a != nil { + actions = append(actions, *a) + } + if a := buildAction4KBAgent(synthesizedComp.LifecycleActions.Reconfigure, "reconfigure"); a != nil { + actions = append(actions, *a) + } + if a := buildAction4KBAgent(synthesizedComp.LifecycleActions.AccountProvision, "accountProvision"); a != nil { + actions = append(actions, *a) + } + + if a, p := buildProbe4KBAgent(synthesizedComp.LifecycleActions.RoleProbe, "roleProbe"); a != nil && p != nil { + actions = append(actions, *a) + probes = append(probes, *p) + } + + return kbagent.BuildStartupEnvs(actions, probes) +} + +func buildAction4KBAgent(handler *appsv1alpha1.LifecycleActionHandler, name string) *proto.Action { + if handler == nil { + return nil + } + return buildAction4KBAgentLow(handler.CustomHandler, name) +} + +func buildAction4KBAgentLow(action *appsv1alpha1.Action, name string) *proto.Action { + if action == nil || action.Exec == nil { + return nil + } + a := &proto.Action{ + Name: name, + Exec: &proto.ExecAction{ + Commands: action.Exec.Command, + Args: action.Exec.Args, + // Env: action.Exec.Env, + }, + TimeoutSeconds: action.TimeoutSeconds, + } + if action.RetryPolicy != nil { + a.RetryPolicy = &proto.RetryPolicy{ + MaxRetries: action.RetryPolicy.MaxRetries, + RetryInterval: action.RetryPolicy.RetryInterval, + } + } + return a +} + +func buildProbe4KBAgent(probe *appsv1alpha1.Probe, name string) (*proto.Action, *proto.Probe) { + if probe == nil || probe.Exec == nil { + return nil, nil + } + a := buildAction4KBAgentLow(&probe.Action, name) + p := &proto.Probe{ + Action: name, + InitialDelaySeconds: probe.InitialDelaySeconds, + PeriodSeconds: probe.PeriodSeconds, + SuccessThreshold: probe.SuccessThreshold, + FailureThreshold: probe.FailureThreshold, + ReportPeriodSeconds: nil, // TODO: impl + } + return a, p +} + +func adaptKBAgentIfCustomImageNContainerDefined(synthesizedComp *SynthesizedComponent, container *corev1.Container) error { + image, c, err := customExecActionImageNContainer(synthesizedComp) + if err != nil { + return err + } + if len(image) == 0 { + return nil + } + + // init-container to copy binaries to the shared mount point /kubeblocks + initContainer := buildKBAgentInitContainer() + synthesizedComp.PodSpec.InitContainers = append(synthesizedComp.PodSpec.InitContainers, *initContainer) + + container.Image = image + container.Command[0] = kbAgentCommandOnSharedMount + container.VolumeMounts = append(container.VolumeMounts, sharedVolumeMount) + + // TODO: share more container resources + if c != nil { + container.VolumeMounts = append(container.VolumeMounts, c.VolumeMounts...) + } + + return nil +} + +func customExecActionImageNContainer(synthesizedComp *SynthesizedComponent) (string, *corev1.Container, error) { + if synthesizedComp.LifecycleActions == nil { + return "", nil, nil + } + + handlers := []*appsv1alpha1.LifecycleActionHandler{ + synthesizedComp.LifecycleActions.PostProvision, + synthesizedComp.LifecycleActions.PreTerminate, + synthesizedComp.LifecycleActions.MemberJoin, + synthesizedComp.LifecycleActions.MemberLeave, + synthesizedComp.LifecycleActions.Readonly, + synthesizedComp.LifecycleActions.Readwrite, + synthesizedComp.LifecycleActions.DataDump, + synthesizedComp.LifecycleActions.DataLoad, + synthesizedComp.LifecycleActions.Reconfigure, + synthesizedComp.LifecycleActions.AccountProvision, + } + if synthesizedComp.LifecycleActions.RoleProbe != nil && synthesizedComp.LifecycleActions.RoleProbe.Exec != nil { + handlers = append(handlers, &appsv1alpha1.LifecycleActionHandler{ + CustomHandler: &synthesizedComp.LifecycleActions.RoleProbe.Action, + }) + } + + var image, container string + for _, handler := range handlers { + if handler == nil || handler.CustomHandler == nil || handler.CustomHandler.Exec == nil { + continue + } + if handler.CustomHandler.Exec.Image != "" { + if len(image) > 0 && image != handler.CustomHandler.Exec.Image { + return "", nil, fmt.Errorf("only one exec image is allowed in lifecycle actions") + } + image = handler.CustomHandler.Exec.Image + } + if handler.CustomHandler.Exec.Container != "" { + if len(container) > 0 && container != handler.CustomHandler.Exec.Container { + return "", nil, fmt.Errorf("only one exec container is allowed in lifecycle actions") + } + container = handler.CustomHandler.Exec.Container + } + } + + var c *corev1.Container + if len(container) > 0 { + for i, cc := range synthesizedComp.PodSpec.Containers { + if cc.Name == container { + c = &synthesizedComp.PodSpec.Containers[i] + break + } + } + if c == nil { + return "", nil, fmt.Errorf("exec container %s not found", container) + } + } + if len(image) > 0 && len(container) > 0 { + if c.Image == image { + return image, c, nil + } + return "", nil, fmt.Errorf("exec image and container must be the same") + } + if len(image) == 0 && len(container) > 0 { + image = c.Image + } + return image, c, nil +} + +func buildKBAgentInitContainer() *corev1.Container { + return builder.NewContainerBuilder(kbAgentInitContainerName). + SetImage(viper.GetString(constant.KBToolsImage)). + SetImagePullPolicy(corev1.PullIfNotPresent). + AddCommands([]string{"cp", "-r", kbAgentCommand, "/bin/curl", kbAgentSharedMountPath + "/"}...). + AddVolumeMounts(sharedVolumeMount). + GetObject() +} + +func getAvailablePorts(containers []corev1.Container, containerPorts []int32) ([]int32, error) { + inUse, err := getInUsePorts(containers) + if err != nil { + return nil, err + } + availablePort := make([]int32, len(containerPorts)) + for i, p := range containerPorts { + if availablePort[i], err = iterAvailablePort(p, inUse); err != nil { + return nil, err + } + } + return availablePort, nil +} + +func getInUsePorts(containers []corev1.Container) (map[int32]bool, error) { + inUse := map[int32]bool{} + for _, container := range containers { + for _, v := range container.Ports { + _, ok := inUse[v.ContainerPort] + if ok { + return nil, fmt.Errorf("containerPorts conflict: [%+v]", v.ContainerPort) + } + inUse[v.ContainerPort] = true + } + } + return inUse, nil +} + +func iterAvailablePort(port int32, set map[int32]bool) (int32, error) { + if port < minAvailablePort || port > maxAvailablePort { + port = minAvailablePort + } + sentinel := port + for { + if _, ok := set[port]; !ok { + set[port] = true + return port, nil + } + port++ + if port == sentinel { + return -1, errors.New("no available port for container") + } + if port > maxAvailablePort { + port = minAvailablePort + } + } +} diff --git a/pkg/controller/component/lifecycle/errors.go b/pkg/controller/component/lifecycle/errors.go new file mode 100644 index 00000000000..9a58b062d05 --- /dev/null +++ b/pkg/controller/component/lifecycle/errors.go @@ -0,0 +1,42 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package lifecycle + +import ( + "errors" +) + +var ( + ErrActionNotDefined = errors.New("action is not defined") + ErrActionNotImplemented = errors.New("action is not implemented") + ErrActionInProgress = errors.New("action is in progress") + ErrActionBusy = errors.New("action is busy") + ErrActionTimeout = errors.New("action timeout") + ErrActionFailed = errors.New("action failed") + ErrActionCanceled = errors.New("action canceled") + ErrActionInternalError = errors.New("action internal error") +) + +func IgnoreNotDefined(err error) error { + if err == ErrActionNotDefined { + return nil + } + return err +} diff --git a/pkg/controller/component/lifecycle/kbagent.go b/pkg/controller/component/lifecycle/kbagent.go new file mode 100644 index 00000000000..1223f10d415 --- /dev/null +++ b/pkg/controller/component/lifecycle/kbagent.go @@ -0,0 +1,235 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package lifecycle + +import ( + "context" + "fmt" + "math/rand" + + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + "github.com/apecloud/kubeblocks/pkg/constant" + "github.com/apecloud/kubeblocks/pkg/controller/component" + kbacli "github.com/apecloud/kubeblocks/pkg/kbagent/client" + "github.com/apecloud/kubeblocks/pkg/kbagent/proto" + "github.com/apecloud/kubeblocks/pkg/kbagent/service" +) + +type lifecycleAction interface { + name() string + parameters(ctx context.Context, cli client.Reader) (map[string]string, error) +} + +type kbagent struct { + synthesizedComp *component.SynthesizedComponent + lifecycleActions *appsv1alpha1.ComponentLifecycleActions + pods []*corev1.Pod + pod *corev1.Pod +} + +var _ Lifecycle = &kbagent{} + +func (a *kbagent) PostProvision(ctx context.Context, cli client.Reader, opts *Options) error { + la := &postProvision{} + return a.checkedCallAction(ctx, cli, a.lifecycleActions.PostProvision, la, opts) +} + +func (a *kbagent) PreTerminate(ctx context.Context, cli client.Reader, opts *Options) error { + la := &preTerminate{} + return a.checkedCallAction(ctx, cli, a.lifecycleActions.PreTerminate, la, opts) +} + +func (a *kbagent) Switchover(ctx context.Context, cli client.Reader, opts *Options) error { + la := &switchover{} + if a.lifecycleActions.Switchover == nil { + return errors.Wrap(ErrActionNotDefined, la.name()) + } + return a.callAction(ctx, cli, a.lifecycleActions.Switchover, la, opts) +} + +func (a *kbagent) MemberJoin(ctx context.Context, cli client.Reader, opts *Options) error { + la := &memberJoin{ + synthesizedComp: a.synthesizedComp, + pod: a.pod, + } + return a.checkedCallAction(ctx, cli, a.lifecycleActions.MemberJoin, la, opts) +} + +func (a *kbagent) MemberLeave(ctx context.Context, cli client.Reader, opts *Options) error { + la := &memberLeave{ + synthesizedComp: a.synthesizedComp, + pod: a.pod, + } + return a.checkedCallAction(ctx, cli, a.lifecycleActions.MemberLeave, la, opts) +} + +func (a *kbagent) DataDump(ctx context.Context, cli client.Reader, opts *Options) error { + la := &dataDump{} + return a.checkedCallAction(ctx, cli, a.lifecycleActions.DataDump, la, opts) +} + +func (a *kbagent) DataLoad(ctx context.Context, cli client.Reader, opts *Options) error { + la := &dataLoad{} + return a.checkedCallAction(ctx, cli, a.lifecycleActions.DataLoad, la, opts) +} + +func (a *kbagent) AccountProvision(ctx context.Context, cli client.Reader, opts *Options, args ...any) error { + la := &accountProvision{args: args} + return a.checkedCallAction(ctx, cli, a.lifecycleActions.AccountProvision, la, opts) +} + +func (a *kbagent) checkedCallAction(ctx context.Context, cli client.Reader, + handler *appsv1alpha1.LifecycleActionHandler, la lifecycleAction, opts *Options) error { + if handler == nil || handler.CustomHandler == nil { + return errors.Wrap(ErrActionNotDefined, la.name()) + } + return a.callAction(ctx, cli, handler.CustomHandler, la, opts) +} + +func (a *kbagent) callAction(ctx context.Context, cli client.Reader, spec *appsv1alpha1.Action, la lifecycleAction, opts *Options) error { + req, err1 := a.buildActionRequest(ctx, cli, la, opts) + if err1 != nil { + return err1 + } + return a.callActionWithSelector(ctx, spec, la, req) +} + +func (a *kbagent) buildActionRequest(ctx context.Context, cli client.Reader, la lifecycleAction, opts *Options) (*proto.ActionRequest, error) { + parameters, err := la.parameters(ctx, cli) + if err != nil { + return nil, err + } + + req := &proto.ActionRequest{ + Action: la.name(), + Parameters: parameters, + } + if opts != nil { + if opts.NonBlocking != nil { + req.NonBlocking = opts.NonBlocking + } + if opts.TimeoutSeconds != nil { + req.TimeoutSeconds = opts.TimeoutSeconds + } + if opts.RetryPolicy != nil { + req.RetryPolicy = &proto.RetryPolicy{ + MaxRetries: opts.RetryPolicy.MaxRetries, + RetryInterval: opts.RetryPolicy.RetryInterval, + } + } + } + return req, nil +} + +func (a *kbagent) callActionWithSelector(ctx context.Context, spec *appsv1alpha1.Action, la lifecycleAction, req *proto.ActionRequest) error { + pods, err := a.selectTargetPods(spec) + if err != nil { + return err + } + if len(pods) == 0 { + return fmt.Errorf("no available pod to call action %s", la.name()) + } + + // TODO: impl + // - back-off to retry + // - timeout + for _, pod := range a.pods { + cli, err1 := kbacli.NewClient(*pod) + if err1 != nil { + return err1 + } + if cli == nil { + continue // not defined, for test only + } + _, err2 := cli.CallAction(ctx, *req) + if err2 != nil { + return a.error2(la, err2) + } + } + return nil +} + +func (a *kbagent) selectTargetPods(spec *appsv1alpha1.Action) ([]*corev1.Pod, error) { + if spec.Exec == nil || len(spec.Exec.TargetPodSelector) == 0 { + return []*corev1.Pod{a.pod}, nil + } + + anyPod := func() []*corev1.Pod { + i := rand.Int() % len(a.pods) + return []*corev1.Pod{a.pods[i]} + } + + allPods := func() []*corev1.Pod { + return a.pods + } + + podsWithRole := func() []*corev1.Pod { + roleName := spec.Exec.MatchingKey + var pods []*corev1.Pod + for i, pod := range a.pods { + if len(pod.Labels) != 0 { + if pod.Labels[constant.RoleLabelKey] == roleName { + pods = append(pods, a.pods[i]) + } + } + } + return pods + } + + switch spec.Exec.TargetPodSelector { + case appsv1alpha1.AnyReplica: + return anyPod(), nil + case appsv1alpha1.AllReplicas: + return allPods(), nil + case appsv1alpha1.RoleSelector: + return podsWithRole(), nil + case appsv1alpha1.OrdinalSelector: + return nil, fmt.Errorf("ordinal selector is not supported") + default: + return nil, fmt.Errorf("unknown pod selector: %s", spec.Exec.TargetPodSelector) + } +} + +func (a *kbagent) error2(la lifecycleAction, err error) error { + switch { + case err == nil: + return nil + case errors.Is(err, service.ErrNotDefined): + return errors.Wrap(ErrActionNotDefined, la.name()) + case errors.Is(err, service.ErrNotImplemented): + return errors.Wrap(ErrActionNotImplemented, la.name()) + case errors.Is(err, service.ErrInProgress): + return errors.Wrap(ErrActionInProgress, la.name()) + case errors.Is(err, service.ErrBusy): + return errors.Wrap(ErrActionBusy, la.name()) + case errors.Is(err, service.ErrTimeout): + return errors.Wrap(ErrActionTimeout, la.name()) + case errors.Is(err, service.ErrFailed): + return errors.Wrap(ErrActionFailed, la.name()) + case errors.Is(err, service.ErrInternalError): + return errors.Wrap(ErrActionInternalError, la.name()) + default: + return err + } +} diff --git a/pkg/kb_agent/httpserver/errors.go b/pkg/controller/component/lifecycle/lfa_account.go similarity index 65% rename from pkg/kb_agent/httpserver/errors.go rename to pkg/controller/component/lifecycle/lfa_account.go index 1cbf2edf057..d4cfea8b1f0 100644 --- a/pkg/kb_agent/httpserver/errors.go +++ b/pkg/controller/component/lifecycle/lfa_account.go @@ -17,18 +17,24 @@ You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ -package httpserver +package lifecycle -// ErrorResponse is an HTTP response message sent back to calling clients. -type ErrorResponse struct { - ErrorCode string `json:"errorCode"` - Message string `json:"message"` +import ( + "context" + + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type accountProvision struct { + args any +} + +var _ lifecycleAction = &accountProvision{} + +func (a *accountProvision) name() string { + return "accountProvision" } -// NewErrorResponse returns a new ErrorResponse. -func NewErrorResponse(errorCode, message string) ErrorResponse { - return ErrorResponse{ - ErrorCode: errorCode, - Message: message, - } +func (a *accountProvision) parameters(ctx context.Context, cli client.Reader) (map[string]string, error) { + return nil, nil } diff --git a/pkg/kb_agent/handlers/errors.go b/pkg/controller/component/lifecycle/lfa_common.go similarity index 82% rename from pkg/kb_agent/handlers/errors.go rename to pkg/controller/component/lifecycle/lfa_common.go index c1f6ef84ae9..e1b542d704c 100644 --- a/pkg/kb_agent/handlers/errors.go +++ b/pkg/controller/component/lifecycle/lfa_common.go @@ -17,14 +17,4 @@ You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ -package handlers - -import "fmt" - -const ( - errMsgNotImplemented = "not implemented" -) - -var ( - ErrNotImplemented = fmt.Errorf("%s", errMsgNotImplemented) -) +package lifecycle diff --git a/pkg/kb_agent/handlers/grpc_handler.go b/pkg/controller/component/lifecycle/lfa_component.go similarity index 51% rename from pkg/kb_agent/handlers/grpc_handler.go rename to pkg/controller/component/lifecycle/lfa_component.go index 25c913d9570..bc738725167 100644 --- a/pkg/kb_agent/handlers/grpc_handler.go +++ b/pkg/controller/component/lifecycle/lfa_component.go @@ -17,37 +17,42 @@ You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ -package handlers +package lifecycle import ( "context" - "github.com/go-logr/logr" - "github.com/pkg/errors" - ctrl "sigs.k8s.io/controller-runtime" - - "github.com/apecloud/kubeblocks/pkg/kb_agent/util" + "sigs.k8s.io/controller-runtime/pkg/client" ) -type GRPCHandler struct { - Logger logr.Logger +type postProvision struct { + // namespace string + // clusterName string + // compName string +} + +var _ lifecycleAction = &postProvision{} + +func (a *postProvision) name() string { + return "postProvision" } -var _ Handler = &GRPCHandler{} +func (a *postProvision) parameters(ctx context.Context, cli client.Reader) (map[string]string, error) { + return nil, nil +} + +type preTerminate struct { + // namespace string + // clusterName string + // compName string +} -func NewGRPCHandler(properties map[string]string) (*GRPCHandler, error) { - logger := ctrl.Log.WithName("GRPC handler") - h := &GRPCHandler{ - Logger: logger, - } +var _ lifecycleAction = &preTerminate{} - return h, nil +func (a *preTerminate) name() string { + return "preTerminate" } -func (h *GRPCHandler) Do(ctx context.Context, setting util.HandlerSpec, args map[string]any) (*Response, error) { - if setting.GPRC == nil { - return nil, errors.New("grpc setting is nil") - } - // TODO: implement grpc handler - return nil, ErrNotImplemented +func (a *preTerminate) parameters(ctx context.Context, cli client.Reader) (map[string]string, error) { + return nil, nil } diff --git a/pkg/kb_agent/util/command_test.go b/pkg/controller/component/lifecycle/lfa_data.go similarity index 58% rename from pkg/kb_agent/util/command_test.go rename to pkg/controller/component/lifecycle/lfa_data.go index 93388e17cf3..b8566799758 100644 --- a/pkg/kb_agent/util/command_test.go +++ b/pkg/controller/component/lifecycle/lfa_data.go @@ -17,34 +17,34 @@ You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ -package util +package lifecycle import ( "context" - "os" - "testing" - "github.com/stretchr/testify/assert" + "sigs.k8s.io/controller-runtime/pkg/client" ) -func TestExecCommand(t *testing.T) { - // Set up test environment - ctx := context.Background() - command := []string{"binary not exists"} - envs := []string{"ENV_VAR=value"} +type dataDump struct{} - // Call the function - _, err := ExecCommand(ctx, command, envs) +var _ lifecycleAction = &dataDump{} - // Check the results - assert.Error(t, err) +func (a *dataDump) name() string { + return "dataDump" } -func TestGetAllEnvs(t *testing.T) { - args := map[string]any{ - "test": "test", - } - envs := GetAllEnvs(args) - assert.NotNil(t, envs) - assert.Equal(t, len(os.Environ())+1, len(envs)) +func (a *dataDump) parameters(ctx context.Context, cli client.Reader) (map[string]string, error) { + return nil, nil +} + +type dataLoad struct{} + +var _ lifecycleAction = &dataLoad{} + +func (a *dataLoad) name() string { + return "dataLoad" +} + +func (a *dataLoad) parameters(ctx context.Context, cli client.Reader) (map[string]string, error) { + return nil, nil } diff --git a/pkg/controller/component/lifecycle/lfa_member.go b/pkg/controller/component/lifecycle/lfa_member.go new file mode 100644 index 00000000000..a438babfb83 --- /dev/null +++ b/pkg/controller/component/lifecycle/lfa_member.go @@ -0,0 +1,80 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package lifecycle + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/apecloud/kubeblocks/pkg/controller/component" +) + +const ( + joinMemberPodFQDNVar = "KB_JOIN_MEMBER_POD_FQDN" + joinMemberPodNameVar = "KB_JOIN_MEMBER_POD_NAME" + leaveMemberPodFQDNVar = "KB_LEAVE_MEMBER_POD_FQDN" + leaveMemberPodNameVar = "KB_LEAVE_MEMBER_POD_NAME" +) + +type memberJoin struct { + synthesizedComp *component.SynthesizedComponent + pod *corev1.Pod +} + +var _ lifecycleAction = &memberJoin{} + +func (a *memberJoin) name() string { + return "memberJoin" +} + +func (a *memberJoin) parameters(ctx context.Context, cli client.Reader) (map[string]string, error) { + // The container executing this action has access to following environment variables: + // + // - KB_JOIN_MEMBER_POD_FQDN: The pod FQDN of the replica being added to the group. + // - KB_JOIN_MEMBER_POD_NAME: The pod name of the replica being added to the group. + return map[string]string{ + joinMemberPodFQDNVar: component.PodFQDN(a.synthesizedComp.Namespace, a.synthesizedComp.FullCompName, a.pod.Name), + joinMemberPodNameVar: a.pod.Name, + }, nil +} + +type memberLeave struct { + synthesizedComp *component.SynthesizedComponent + pod *corev1.Pod +} + +var _ lifecycleAction = &memberLeave{} + +func (a *memberLeave) name() string { + return "memberLeave" +} + +func (a *memberLeave) parameters(ctx context.Context, cli client.Reader) (map[string]string, error) { + // The container executing this action has access to following environment variables: + // + // - KB_LEAVE_MEMBER_POD_FQDN: The pod name of the replica being removed from the group. + // - KB_LEAVE_MEMBER_POD_NAME: The pod name of the replica being removed from the group. + return map[string]string{ + leaveMemberPodFQDNVar: component.PodFQDN(a.synthesizedComp.Namespace, a.synthesizedComp.FullCompName, a.pod.Name), + leaveMemberPodNameVar: a.pod.Name, + }, nil +} diff --git a/pkg/controller/component/lifecycle/lfa_role.go b/pkg/controller/component/lifecycle/lfa_role.go new file mode 100644 index 00000000000..576fb9d1acb --- /dev/null +++ b/pkg/controller/component/lifecycle/lfa_role.go @@ -0,0 +1,41 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package lifecycle + +import ( + "context" + + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type switchover struct { + // synthesizedComp *component.SynthesizedComponent + // switchover *appsv1alpha1.Switchover +} + +var _ lifecycleAction = &switchover{} + +func (a *switchover) name() string { + return "switchover" +} + +func (a *switchover) parameters(ctx context.Context, cli client.Reader) (map[string]string, error) { + return nil, nil +} diff --git a/pkg/controller/component/lifecycle/lifecycle.go b/pkg/controller/component/lifecycle/lifecycle.go new file mode 100644 index 00000000000..a333bd2fca8 --- /dev/null +++ b/pkg/controller/component/lifecycle/lifecycle.go @@ -0,0 +1,81 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package lifecycle + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + appsv1alpha1 "github.com/apecloud/kubeblocks/apis/apps/v1alpha1" + "github.com/apecloud/kubeblocks/pkg/controller/component" +) + +type Options struct { + NonBlocking *bool + TimeoutSeconds *int32 + RetryPolicy *appsv1alpha1.RetryPolicy +} + +type Lifecycle interface { + PostProvision(ctx context.Context, cli client.Reader, opts *Options) error + + PreTerminate(ctx context.Context, cli client.Reader, opts *Options) error + + // RoleProbe(ctx context.Context, cli client.Reader, opts *Options) ([]byte, error) + + Switchover(ctx context.Context, cli client.Reader, opts *Options) error + + MemberJoin(ctx context.Context, cli client.Reader, opts *Options) error + + MemberLeave(ctx context.Context, cli client.Reader, opts *Options) error + + // Readonly(ctx context.Context, cli client.Reader, opts *Options) error + + // Readwrite(ctx context.Context, cli client.Reader, opts *Options) error + + DataDump(ctx context.Context, cli client.Reader, opts *Options) error + + DataLoad(ctx context.Context, cli client.Reader, opts *Options) error + + // Reconfigure(ctx context.Context, cli client.Reader, opts *Options) error + + AccountProvision(ctx context.Context, cli client.Reader, opts *Options, args ...any) error +} + +func New(synthesizedComp *component.SynthesizedComponent, pod *corev1.Pod, pods ...*corev1.Pod) (Lifecycle, error) { + if pod == nil && len(pods) == 0 { + return nil, fmt.Errorf("either pod or pods must be provided to call lifecycle actions") + } + if pod == nil { + pod = pods[0] + } + if len(pods) == 0 { + pods = []*corev1.Pod{pod} + } + return &kbagent{ + synthesizedComp: synthesizedComp, + lifecycleActions: synthesizedComp.LifecycleActions, + pods: pods, + pod: pod, + }, nil +} diff --git a/pkg/controller/component/synthesize_component.go b/pkg/controller/component/synthesize_component.go index 7d23833e797..01aad7d4f6e 100644 --- a/pkg/controller/component/synthesize_component.go +++ b/pkg/controller/component/synthesize_component.go @@ -210,10 +210,8 @@ func buildSynthesizedComponent(reqCtx intctrlutil.RequestCtx, // build runtimeClassName buildRuntimeClassName(synthesizeComp, comp) - // build lorryContainer - // TODO(xingran): buildLorryContainers relies on synthesizeComp.CharacterType, which will be deprecated in the future. - if err := buildLorryContainers(reqCtx, synthesizeComp, clusterCompSpec); err != nil { - reqCtx.Log.Error(err, "build lorry containers failed.") + if err = buildKBAgentContainer(synthesizeComp); err != nil { + reqCtx.Log.Error(err, "build kb-agent container failed") return nil, err } diff --git a/pkg/controller/component/vars.go b/pkg/controller/component/vars.go index 18a47a7ab6d..ce9f7553ea1 100644 --- a/pkg/controller/component/vars.go +++ b/pkg/controller/component/vars.go @@ -1178,18 +1178,12 @@ func resolveComponentPodFQDNsRef(ctx context.Context, cli client.Reader, synthes for i := range comp.Spec.Instances { templates = append(templates, &comp.Spec.Instances[i]) } - clusterDomainFn := func(name string) string { - return fmt.Sprintf("%s.%s", name, viper.GetString(constant.KubernetesClusterDomainEnv)) - } names, err := instanceset.GenerateAllInstanceNames(comp.Name, comp.Spec.Replicas, templates, comp.Spec.OfflineInstances, workloads.Ordinals{}) if err != nil { return nil, nil, err } - fqdn := func(name string) string { - return clusterDomainFn(fmt.Sprintf("%s.%s-headless.%s.svc", name, comp.Name, synthesizedComp.Namespace)) - } for i := range names { - names[i] = fqdn(names[i]) + names[i] = PodFQDN(synthesizedComp.Namespace, comp.Name, names[i]) } return &corev1.EnvVar{Name: defineKey, Value: strings.Join(names, ",")}, nil, nil } diff --git a/pkg/controller/component/workload_utils.go b/pkg/controller/component/workload_utils.go index b23fe9abf61..8e5b15e3de7 100644 --- a/pkg/controller/component/workload_utils.go +++ b/pkg/controller/component/workload_utils.go @@ -35,6 +35,7 @@ import ( "github.com/apecloud/kubeblocks/pkg/constant" "github.com/apecloud/kubeblocks/pkg/controller/instanceset" "github.com/apecloud/kubeblocks/pkg/generics" + viper "github.com/apecloud/kubeblocks/pkg/viperx" ) func ListOwnedWorkloads(ctx context.Context, cli client.Reader, namespace, clusterName, compName string) ([]*workloads.InstanceSet, error) { @@ -177,3 +178,11 @@ func GetTemplateNameAndOrdinal(workloadName, podName string) (string, int32, err } return templateName, int32(index), nil } + +func PodFQDN(namespace, compName, podName string) string { + return fmt.Sprintf("%s.%s-headless.%s.svc.%s", podName, compName, namespace, clusterDomain()) +} + +func clusterDomain() string { + return viper.GetString(constant.KubernetesClusterDomainEnv) +} diff --git a/pkg/controller/factory/builder_test.go b/pkg/controller/factory/builder_test.go index 889e250e36d..3e4fef47caf 100644 --- a/pkg/controller/factory/builder_test.go +++ b/pkg/controller/factory/builder_test.go @@ -181,7 +181,7 @@ var _ = Describe("builder", func() { Expect(its.Spec.Roles).Should(HaveLen(len(compDef.Spec.Roles))) // test role probe - Expect(its.Spec.RoleProbe).ShouldNot(BeNil()) + Expect(its.Spec.RoleProbe).Should(BeNil()) // test member update strategy Expect(its.Spec.MemberUpdateStrategy).ShouldNot(BeNil()) diff --git a/pkg/controller/instanceset/pod_role_event_handler.go b/pkg/controller/instanceset/pod_role_event_handler.go index d6c8e1b00d1..5f32e03cf46 100644 --- a/pkg/controller/instanceset/pod_role_event_handler.go +++ b/pkg/controller/instanceset/pod_role_event_handler.go @@ -27,6 +27,7 @@ import ( "strconv" "strings" + "github.com/go-logr/logr" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" @@ -37,6 +38,7 @@ import ( "github.com/apecloud/kubeblocks/pkg/constant" "github.com/apecloud/kubeblocks/pkg/controller/multicluster" intctrlutil "github.com/apecloud/kubeblocks/pkg/controllerutil" + "github.com/apecloud/kubeblocks/pkg/kbagent/proto" "github.com/apecloud/kubeblocks/pkg/lorry/util" ) @@ -65,6 +67,9 @@ const ( var roleMessageRegex = regexp.MustCompile(`Readiness probe failed: .*({.*})`) func (h *PodRoleEventHandler) Handle(cli client.Client, reqCtx intctrlutil.RequestCtx, recorder record.EventRecorder, event *corev1.Event) error { + // HACK: to support kb-agent probe event + event = h.transformKBAgentProbeEvent(reqCtx.Log, event) + filePaths := []string{readinessProbeEventFieldPath, util.LegacyEventFieldPath, util.LorryEventFieldPath} if !slices.Contains(filePaths, event.InvolvedObject.FieldPath) || event.Reason != string(util.CheckRoleOperation) { return nil @@ -92,6 +97,32 @@ func (h *PodRoleEventHandler) Handle(cli client.Client, reqCtx intctrlutil.Reque return cli.Patch(reqCtx.Ctx, event, patch, inDataContextUnspecified()) } +func (h *PodRoleEventHandler) transformKBAgentProbeEvent(logger logr.Logger, event *corev1.Event) *corev1.Event { + if event.ReportingController != "kbagent" || event.Reason != "roleProbe" { + return event + } + + probeEvent := &proto.ProbeEvent{} + if err := json.Unmarshal([]byte(event.Message), probeEvent); err != nil { + logger.Error(err, "unmarshal probe event message failed") + return event + } + + message := &probeMessage{ + Message: probeEvent.Message, + Role: strings.TrimSpace(string(probeEvent.Output)), + } + if probeEvent.Code == 0 { + message.Event = successEvent + } + data, _ := json.Marshal(message) + + event.InvolvedObject.FieldPath = util.LorryEventFieldPath + event.Reason = string(util.CheckRoleOperation) + event.Message = string(data) + return event +} + // handleRoleChangedEvent handles role changed event and return role. func handleRoleChangedEvent(cli client.Client, reqCtx intctrlutil.RequestCtx, _ record.EventRecorder, event *corev1.Event) (string, error) { // parse probe event message diff --git a/pkg/controllerutil/pod_utils.go b/pkg/controllerutil/pod_utils.go index be2d4c9efaa..ec94ce55269 100644 --- a/pkg/controllerutil/pod_utils.go +++ b/pkg/controllerutil/pod_utils.go @@ -333,6 +333,19 @@ func GetPortByPortName(containers []corev1.Container, portName string) (int32, e return 0, fmt.Errorf("port %s not found", portName) } +func GetPortByName(pod corev1.Pod, cname, pname string) (int32, error) { + for _, container := range pod.Spec.Containers { + if container.Name == cname { + for _, port := range container.Ports { + if port.Name == pname { + return port.ContainerPort, nil + } + } + } + } + return 0, fmt.Errorf("port %s not found", pname) +} + func GetLorryGRPCPort(pod *corev1.Pod) (int32, error) { return GetLorryGRPCPortFromContainers(pod.Spec.Containers) } diff --git a/pkg/kb_agent/cronjobs/checkrole.go b/pkg/kb_agent/cronjobs/checkrole.go deleted file mode 100644 index 1b600c95a55..00000000000 --- a/pkg/kb_agent/cronjobs/checkrole.go +++ /dev/null @@ -1,93 +0,0 @@ -/* -Copyright (C) 2022-2024 ApeCloud Co., Ltd - -This file is part of KubeBlocks project - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ - -package cronjobs - -import ( - "context" - "time" - - "github.com/spf13/pflag" - - "github.com/apecloud/kubeblocks/pkg/kb_agent/handlers" - "github.com/apecloud/kubeblocks/pkg/kb_agent/util" -) - -type CheckRoleJob struct { - CommonJob - lastRole string - roleUnchangedEventCount int -} - -var sendRoleEventPeriodically bool -var sendRoleEventFrequency int - -func init() { - pflag.BoolVar(&sendRoleEventPeriodically, "send-role-event-periodically", false, "Enable the mechanism to send role events periodically to prevent event loss.") - pflag.IntVar(&sendRoleEventFrequency, "send-role-event-frequency", 300, "the frequency of sending role events.") -} - -func NewCheckRoleJob(commonJob CommonJob) *CheckRoleJob { - checkRoleJob := &CheckRoleJob{ - CommonJob: commonJob, - lastRole: "waitForStart", - } - - checkRoleJob.Do = checkRoleJob.do - return checkRoleJob -} - -func (job *CheckRoleJob) do() error { - ctx1, cancel := context.WithTimeout(context.Background(), time.Duration(job.TimeoutSeconds)) - defer cancel() - resp, err := handlers.Do(ctx1, job.Name, nil) - - if err != nil { - return err - } - - role := resp.Message - if job.lastRole == role { - if !sendRoleEventPeriodically { - return nil - } - job.roleUnchangedEventCount++ - if job.roleUnchangedEventCount%sendRoleEventFrequency != 0 { - return nil - } - logger.Info("send role event periodically", "role", role) - } else { - job.roleUnchangedEventCount = 0 - logger.Info("send role changed event", "role", role) - } - - result := util.RoleProbeMessage{ - MessageBase: util.MessageBase{ - Event: util.OperationSuccess, - Action: job.Name, - }, - Role: role, - } - err = util.SentEventForProbe(context.Background(), result) - if err != nil { - return err - } - job.lastRole = role - return nil -} diff --git a/pkg/kb_agent/cronjobs/checkrole_test.go b/pkg/kb_agent/cronjobs/checkrole_test.go deleted file mode 100644 index 48ae5786a0f..00000000000 --- a/pkg/kb_agent/cronjobs/checkrole_test.go +++ /dev/null @@ -1,133 +0,0 @@ -/* -Copyright (C) 2022-2024 ApeCloud Co., Ltd - -This file is part of KubeBlocks project - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ - -package cronjobs - -import ( - "context" - "encoding/json" - "errors" - "testing" - - "github.com/spf13/viper" - "github.com/stretchr/testify/assert" - - "github.com/apecloud/kubeblocks/pkg/constant" - "github.com/apecloud/kubeblocks/pkg/kb_agent/handlers" - "github.com/apecloud/kubeblocks/pkg/kb_agent/util" -) - -func TestCheckRoleJob(t *testing.T) { - actionHandlerSpecs := map[string]util.HandlerSpec{ - constant.RoleProbeAction: {}, - } - actionJSON, _ := json.Marshal(actionHandlerSpecs) - viper.Set(constant.KBEnvActionHandlers, string(actionJSON)) - assert.Nil(t, handlers.InitHandlers()) - - commonJob := CommonJob{ - Name: constant.RoleProbeAction, - TimeoutSeconds: 10, - } - job := NewCheckRoleJob(commonJob) - - t.Run("do - role unchanged with sendRoleEventPeriodically disable", func(t *testing.T) { - job.lastRole = "role1" - job.roleUnchangedEventCount = 0 - - handler := &MockHandler{} - handler.DoFunc = func(ctx context.Context, setting util.HandlerSpec, args map[string]interface{}) (*handlers.Response, error) { - return &handlers.Response{ - Message: "role1", - }, nil - } - handlers.SetDefaultHandler(handler) - - err := job.do() - - assert.NoError(t, err) - assert.Equal(t, 0, job.roleUnchangedEventCount) - }) - - t.Run("do - role unchanged with sendRoleEventPeriodically enable", func(t *testing.T) { - job.lastRole = "role1" - job.roleUnchangedEventCount = 0 - sendRoleEventPeriodically = true - - handler := &MockHandler{} - handler.DoFunc = func(ctx context.Context, setting util.HandlerSpec, args map[string]interface{}) (*handlers.Response, error) { - return &handlers.Response{ - Message: "role1", - }, nil - } - handlers.SetDefaultHandler(handler) - - err := job.do() - - assert.NoError(t, err) - assert.Equal(t, 1, job.roleUnchangedEventCount) - }) - - t.Run("do - role changed", func(t *testing.T) { - job.lastRole = "role1" - job.roleUnchangedEventCount = 0 - - handler := &MockHandler{ - DoFunc: func(ctx context.Context, setting util.HandlerSpec, args map[string]interface{}) (*handlers.Response, error) { - return &handlers.Response{ - Message: "role2", - }, nil - }, - } - handlers.SetDefaultHandler(handler) - - err := job.do() - - assert.NoError(t, err) - assert.Equal(t, 0, job.roleUnchangedEventCount) - }) - - t.Run("do - error", func(t *testing.T) { - job.lastRole = "role1" - job.roleUnchangedEventCount = 0 - - handler := &MockHandler{ - DoFunc: func(ctx context.Context, setting util.HandlerSpec, args map[string]interface{}) (*handlers.Response, error) { - return nil, errors.New("some error") - }, - } - handlers.SetDefaultHandler(handler) - - err := job.do() - - assert.Error(t, err) - assert.Equal(t, 0, job.roleUnchangedEventCount) - }) -} - -type MockHandler struct { - DoFunc func(ctx context.Context, setting util.HandlerSpec, args map[string]interface{}) (*handlers.Response, error) -} - -func (h *MockHandler) Do(ctx context.Context, setting util.HandlerSpec, args map[string]interface{}) (*handlers.Response, error) { - if h.DoFunc != nil { - return h.DoFunc(ctx, setting, args) - } - return nil, handlers.ErrNotImplemented -} diff --git a/pkg/kb_agent/cronjobs/job.go b/pkg/kb_agent/cronjobs/job.go deleted file mode 100644 index 3ea35411f6a..00000000000 --- a/pkg/kb_agent/cronjobs/job.go +++ /dev/null @@ -1,106 +0,0 @@ -/* -Copyright (C) 2022-2024 ApeCloud Co., Ltd - -This file is part of KubeBlocks project - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ - -package cronjobs - -import ( - "context" - "fmt" - "time" - - "github.com/apecloud/kubeblocks/pkg/constant" - "github.com/apecloud/kubeblocks/pkg/kb_agent/util" -) - -type Job interface { - Start() - Stop() -} - -type CommonJob struct { - Name string - Ticker *time.Ticker - TimeoutSeconds int - PeriodSeconds int - SuccessThreshold int - FailureThreshold int - FailedCount int - ReportFrequency int - Do func() error -} - -func NewJob(name string, cronJob *util.CronJob) (Job, error) { - job := &CommonJob{ - Name: name, - TimeoutSeconds: 60, - PeriodSeconds: 60, - SuccessThreshold: 1, - FailureThreshold: 3, - ReportFrequency: 60, - } - - if cronJob.PeriodSeconds != 0 { - job.PeriodSeconds = cronJob.PeriodSeconds - } - - if cronJob.SuccessThreshold != 0 { - job.SuccessThreshold = cronJob.SuccessThreshold - } - - if cronJob.FailureThreshold != 0 { - job.FailureThreshold = cronJob.FailureThreshold - } - - if cronJob.ReportFrequency != 0 { - job.ReportFrequency = cronJob.ReportFrequency - } - - if name == constant.RoleProbeAction { - return NewCheckRoleJob(*job), nil - } - - return nil, fmt.Errorf("%s not implemented", name) -} - -func (job *CommonJob) Start() { - job.Ticker = time.NewTicker(time.Duration(job.PeriodSeconds) * time.Second) - defer job.Ticker.Stop() - for range job.Ticker.C { - err := job.Do() - if err != nil { - logger.Info("Failed to run job", "name", job.Name, "error", err.Error()) - if job.FailedCount%job.ReportFrequency == 0 { - logger.Info("job failed continuously", "name", job.Name, "times", job.FailedCount) - msg := util.MessageBase{ - Event: util.OperationFailed, - Action: job.Name, - Message: err.Error(), - } - _ = util.SentEventForProbe(context.Background(), msg) - } - job.FailedCount++ - } else { - job.FailedCount = 0 - } - } -} - -func (job *CommonJob) Stop() { - job.Ticker.Stop() -} diff --git a/pkg/kb_agent/cronjobs/job_test.go b/pkg/kb_agent/cronjobs/job_test.go deleted file mode 100644 index 7d612bb1814..00000000000 --- a/pkg/kb_agent/cronjobs/job_test.go +++ /dev/null @@ -1,99 +0,0 @@ -/* -Copyright (C) 2022-2024 ApeCloud Co., Ltd - -This file is part of KubeBlocks project - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ - -package cronjobs - -import ( - "fmt" - "testing" - "time" - - "github.com/stretchr/testify/assert" - - "github.com/apecloud/kubeblocks/pkg/constant" - "github.com/apecloud/kubeblocks/pkg/kb_agent/util" -) - -func TestNewJob(t *testing.T) { - t.Run("NewJob", func(t *testing.T) { - cronJob := &util.CronJob{ - PeriodSeconds: 1, - SuccessThreshold: 2, - FailureThreshold: 2, - ReportFrequency: 2, - } - name := constant.RoleProbeAction - job, err := NewJob(name, cronJob) - assert.NotNil(t, job) - assert.Nil(t, err) - }) - - t.Run("NewJob", func(t *testing.T) { - cronJob := &util.CronJob{ - PeriodSeconds: 0, - SuccessThreshold: 0, - FailureThreshold: 0, - ReportFrequency: 0, - } - name := "test" - job, err := NewJob(name, cronJob) - assert.Nil(t, job) - assert.NotNil(t, err) - assert.Error(t, err, fmt.Errorf("%s not implemented", name)) - }) - -} - -func TestCommonJob_Start(t *testing.T) { - t.Run("No errors after startup", func(t *testing.T) { - job := &CommonJob{ - Name: "test", - PeriodSeconds: 1, - ReportFrequency: 1, - FailedCount: 0, - Do: func() error { - return nil - }, - } - go func() { - job.Start() - }() - time.Sleep(3 * time.Second) - job.Ticker.Stop() - assert.Equal(t, 0, job.FailedCount) - }) - - t.Run("Error after startup", func(t *testing.T) { - job := &CommonJob{ - Name: "test", - PeriodSeconds: 2, - ReportFrequency: 1, - FailedCount: 0, - Do: func() error { - return fmt.Errorf("test error") - }, - } - go func() { - job.Start() - }() - time.Sleep(3 * time.Second) - job.Stop() - assert.Equal(t, 1, job.FailedCount) - }) -} diff --git a/pkg/kb_agent/cronjobs/manager.go b/pkg/kb_agent/cronjobs/manager.go deleted file mode 100644 index ad8b4d1818a..00000000000 --- a/pkg/kb_agent/cronjobs/manager.go +++ /dev/null @@ -1,58 +0,0 @@ -/* -Copyright (C) 2022-2024 ApeCloud Co., Ltd - -This file is part of KubeBlocks project - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ - -package cronjobs - -import ( - ctrl "sigs.k8s.io/controller-runtime" - - "github.com/apecloud/kubeblocks/pkg/kb_agent/handlers" -) - -type Manager struct { - Jobs map[string]Job -} - -var logger = ctrl.Log.WithName("cronjobs") - -func NewManager() (*Manager, error) { - actionHandlers := handlers.GetHandlerSpecs() - jobs := make(map[string]Job) - for name, handler := range actionHandlers { - if handler.CronJob == nil { - continue - } - logger.Info("cronjob found", "name", name) - job, err := NewJob(name, handler.CronJob) - if err != nil { - logger.Info("Failed to create job", "name", name, "error", err.Error()) - continue - } - jobs[name] = job - } - return &Manager{ - Jobs: jobs, - }, nil -} - -func (m *Manager) Start() { - for _, job := range m.Jobs { - go job.Start() - } -} diff --git a/pkg/kb_agent/cronjobs/manager_test.go b/pkg/kb_agent/cronjobs/manager_test.go deleted file mode 100644 index f765c00af52..00000000000 --- a/pkg/kb_agent/cronjobs/manager_test.go +++ /dev/null @@ -1,56 +0,0 @@ -/* -Copyright (C) 2022-2024 ApeCloud Co., Ltd - -This file is part of KubeBlocks project - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ - -package cronjobs - -import ( - "encoding/json" - "testing" - - "github.com/spf13/viper" - "github.com/stretchr/testify/assert" - - "github.com/apecloud/kubeblocks/pkg/constant" - "github.com/apecloud/kubeblocks/pkg/kb_agent/handlers" - "github.com/apecloud/kubeblocks/pkg/kb_agent/util" -) - -func TestNewManager(t *testing.T) { - handlers.ResetHandlerSpecs() - actionHandlerSpecs := map[string]util.HandlerSpec{ - constant.RoleProbeAction: { - CronJob: &util.CronJob{ - PeriodSeconds: 1, - SuccessThreshold: 2, - FailureThreshold: 2, - ReportFrequency: 2, - }, - }, - "test": {}, - } - actionJSON, _ := json.Marshal(actionHandlerSpecs) - viper.Set(constant.KBEnvActionHandlers, string(actionJSON)) - assert.Nil(t, handlers.InitHandlers()) - t.Run("NewManager", func(t *testing.T) { - manager, err := NewManager() - assert.NotNil(t, manager) - assert.Nil(t, err) - assert.Equal(t, 1, len(manager.Jobs)) - }) -} diff --git a/pkg/kb_agent/handlers/action.go b/pkg/kb_agent/handlers/action.go deleted file mode 100644 index 69bf6307942..00000000000 --- a/pkg/kb_agent/handlers/action.go +++ /dev/null @@ -1,117 +0,0 @@ -/* -Copyright (C) 2022-2024 ApeCloud Co., Ltd - -This file is part of KubeBlocks project - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ - -package handlers - -import ( - "context" - "encoding/json" - "fmt" - - "github.com/pkg/errors" - "github.com/spf13/viper" - ctrl "sigs.k8s.io/controller-runtime" - - "github.com/apecloud/kubeblocks/pkg/constant" - intctrlutil "github.com/apecloud/kubeblocks/pkg/controllerutil" - "github.com/apecloud/kubeblocks/pkg/kb_agent/util" -) - -var actionHandlerSpecs = map[string]util.HandlerSpec{} -var execHandler *ExecHandler -var grpcHandler *GRPCHandler -var defaultHandler Handler -var logger = ctrl.Log.WithName("EXEC handler") - -func InitHandlers() error { - if len(actionHandlerSpecs) != 0 { - return nil - } - actionJSON := viper.GetString(constant.KBEnvActionHandlers) - if actionJSON == "" { - return errors.New("action handlers is not specified") - } - - err := json.Unmarshal([]byte(actionJSON), &actionHandlerSpecs) - if err != nil { - msg := fmt.Sprintf("unmarshal action handlers [%s] failed: %s", actionJSON, err.Error()) - return errors.New(msg) - } - execHandler, err = NewExecHandler(nil) - if err != nil { - return errors.Wrap(err, "new exec handler failed") - } - - grpcHandler, err = NewGRPCHandler(nil) - if err != nil { - return errors.Wrap(err, "new grpc handler failed") - } - return nil -} - -func GetHandlerSpecs() map[string]util.HandlerSpec { - return actionHandlerSpecs -} - -func ResetHandlerSpecs() { - actionHandlerSpecs = map[string]util.HandlerSpec{} -} - -func Do(ctx context.Context, action string, args map[string]any) (*Response, error) { - if action == "" { - return nil, errors.New("action is empty") - } - handlerSpec, ok := actionHandlerSpecs[action] - if !ok { - return nil, errors.New("action handler spec not found") - } - - handler := GetHandler(handlerSpec) - if intctrlutil.IsNil(handler) { - return nil, errors.New("no handler found") - } - - resp, err := handler.Do(ctx, handlerSpec, args) - if err != nil { - logger.Info("action exec failed", "action", action, "handler spec", handlerSpec, "error", err.Error()) - return nil, err - } - - return resp, nil -} - -func GetHandler(handlerSpec util.HandlerSpec) Handler { - if len(handlerSpec.Command) != 0 { - return execHandler - } - - if len(handlerSpec.GPRC) != 0 { - return grpcHandler - } - - if defaultHandler != nil { - return defaultHandler - } - - return nil -} - -func SetDefaultHandler(handler Handler) { - defaultHandler = handler -} diff --git a/pkg/kb_agent/handlers/action_test.go b/pkg/kb_agent/handlers/action_test.go deleted file mode 100644 index 803a4ecd310..00000000000 --- a/pkg/kb_agent/handlers/action_test.go +++ /dev/null @@ -1,123 +0,0 @@ -/* -Copyright (C) 2022-2024 ApeCloud Co., Ltd - -This file is part of KubeBlocks project - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ - -package handlers - -import ( - "context" - "encoding/json" - "testing" - - "github.com/pkg/errors" - "github.com/spf13/viper" - "github.com/stretchr/testify/assert" - - "github.com/apecloud/kubeblocks/pkg/constant" - "github.com/apecloud/kubeblocks/pkg/kb_agent/util" -) - -func TestDo(t *testing.T) { - ctx := context.Background() - - t.Run("action empty", func(t *testing.T) { - resp, err := Do(ctx, "", nil) - - assert.Error(t, err) - assert.Nil(t, resp) - assert.Equal(t, "action is empty", err.Error()) - }) - - t.Run("action handler spec not found", func(t *testing.T) { - resp, err := Do(ctx, "unknown-action", nil) - - assert.Error(t, err) - assert.Nil(t, resp) - assert.Equal(t, "action handler spec not found", err.Error()) - }) - - t.Run("no handler found", func(t *testing.T) { - actionHandlerSpecs := map[string]util.HandlerSpec{ - "action1": {}, - } - actionJSON, _ := json.Marshal(actionHandlerSpecs) - viper.Set(constant.KBEnvActionHandlers, string(actionJSON)) - assert.Nil(t, InitHandlers()) - - resp, err := Do(ctx, "action1", nil) - - assert.Error(t, err) - assert.Nil(t, resp) - assert.Equal(t, "no handler found", err.Error()) - }) - - t.Run("action exec failed", func(t *testing.T) { - actionHandlerSpecs := map[string]util.HandlerSpec{ - "action1": {}, - } - actionJSON, _ := json.Marshal(actionHandlerSpecs) - viper.Set(constant.KBEnvActionHandlers, string(actionJSON)) - assert.Nil(t, InitHandlers()) - - handler := &MockHandler{} - handler.DoFunc = func(ctx context.Context, handlerSpec util.HandlerSpec, args map[string]interface{}) (*Response, error) { - return nil, errors.New("execution failed") - } - SetDefaultHandler(handler) - - resp, err := Do(ctx, "action1", nil) - - assert.Error(t, err) - assert.Nil(t, resp) - assert.Equal(t, "execution failed", err.Error()) - }) - - t.Run("action exec success", func(t *testing.T) { - actionHandlerSpecs := map[string]util.HandlerSpec{ - "action1": {}, - } - actionJSON, _ := json.Marshal(actionHandlerSpecs) - viper.Set(constant.KBEnvActionHandlers, string(actionJSON)) - assert.Nil(t, InitHandlers()) - - handler := &MockHandler{} - handler.DoFunc = func(ctx context.Context, handlerSpec util.HandlerSpec, args map[string]interface{}) (*Response, error) { - return &Response{ - Message: "success", - }, nil - } - SetDefaultHandler(handler) - - resp, err := Do(ctx, "action1", nil) - - assert.NoError(t, err) - assert.NotNil(t, resp) - assert.Equal(t, "success", resp.Message) - }) -} - -type MockHandler struct { - DoFunc func(ctx context.Context, setting util.HandlerSpec, args map[string]interface{}) (*Response, error) -} - -func (h *MockHandler) Do(ctx context.Context, setting util.HandlerSpec, args map[string]interface{}) (*Response, error) { - if h.DoFunc != nil { - return h.DoFunc(ctx, setting, args) - } - return nil, ErrNotImplemented -} diff --git a/pkg/kb_agent/handlers/exec_handler.go b/pkg/kb_agent/handlers/exec_handler.go deleted file mode 100644 index ec878014739..00000000000 --- a/pkg/kb_agent/handlers/exec_handler.go +++ /dev/null @@ -1,77 +0,0 @@ -/* -Copyright (C) 2022-2024 ApeCloud Co., Ltd - -This file is part of KubeBlocks project - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ - -package handlers - -import ( - "context" - "time" - - "github.com/pkg/errors" - - "github.com/go-logr/logr" - ctrl "sigs.k8s.io/controller-runtime" - - "github.com/apecloud/kubeblocks/pkg/kb_agent/util" -) - -type ExecHandler struct { - Logger logr.Logger - - Executor util.Executor -} - -var _ Handler = &ExecHandler{} - -func NewExecHandler(properties map[string]string) (*ExecHandler, error) { - logger := ctrl.Log.WithName("EXEC handler") - - h := &ExecHandler{ - Logger: logger, - Executor: &util.ExecutorImpl{}, - } - - return h, nil -} - -func (h *ExecHandler) Do(ctx context.Context, setting util.HandlerSpec, args map[string]any) (*Response, error) { - if len(setting.Command) == 0 { - h.Logger.Info("action command is empty!") - return nil, nil - } - envs := util.GetAllEnvs(args) - h.Logger.Info("execute action", "commands", setting.Command, "envs", envs) - if setting.TimeoutSeconds > 0 { - timeoutCtx, cancel := context.WithTimeout(ctx, time.Duration(setting.TimeoutSeconds)*time.Second) - defer cancel() - ctx = timeoutCtx - } - - output, err := h.Executor.ExecCommand(ctx, setting.Command, envs) - - if err != nil { - return nil, errors.Wrap(err, "ExecHandler executes action failed") - } - - h.Logger.V(1).Info("execute action", "output", output) - resp := &Response{ - Message: output, - } - return resp, err -} diff --git a/pkg/kb_agent/handlers/exec_handler_test.go b/pkg/kb_agent/handlers/exec_handler_test.go deleted file mode 100644 index ecdadb334cf..00000000000 --- a/pkg/kb_agent/handlers/exec_handler_test.go +++ /dev/null @@ -1,103 +0,0 @@ -/* -Copyright (C) 2022-2024 ApeCloud Co., Ltd - -This file is part of KubeBlocks project - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ - -package handlers - -import ( - "context" - "testing" - - "github.com/pkg/errors" - - "github.com/stretchr/testify/assert" - - "github.com/apecloud/kubeblocks/pkg/kb_agent/util" -) - -func TestNewExecHandler(t *testing.T) { - execHandler, err := NewExecHandler(nil) - assert.NotNil(t, execHandler) - assert.Nil(t, err) -} - -func TestExecHandlerDo(t *testing.T) { - ctx := context.Background() - - t.Run("action command is empty", func(t *testing.T) { - execHandler := &ExecHandler{} - setting := util.HandlerSpec{} - args := map[string]interface{}{} - resp, err := execHandler.Do(ctx, setting, args) - assert.Nil(t, resp) - assert.Nil(t, err) - }) - - t.Run("execute with timeout failed", func(t *testing.T) { - msg := "execute timeout" - mockExecutor := &MockExecutor{ - ExecCommandFunc: func(ctx context.Context, command []string, envs []string) (string, error) { - return msg, errors.New(msg) - }, - } - execHandler := &ExecHandler{ - Executor: mockExecutor, - } - setting := util.HandlerSpec{ - Command: []string{msg}, - TimeoutSeconds: 1, - } - args := map[string]interface{}{} - resp, err := execHandler.Do(ctx, setting, args) - assert.Nil(t, resp) - assert.NotNil(t, err) - assert.ErrorContains(t, err, msg) - - }) - - t.Run("execute success", func(t *testing.T) { - msg := "execute success" - mockExecutor := &MockExecutor{ - ExecCommandFunc: func(ctx context.Context, command []string, envs []string) (string, error) { - return msg, nil - }, - } - execHandler := &ExecHandler{ - Executor: mockExecutor, - } - setting := util.HandlerSpec{ - Command: []string{msg}, - } - args := map[string]interface{}{} - resp, err := execHandler.Do(ctx, setting, args) - assert.NotNil(t, resp) - assert.Equal(t, msg, resp.Message) - assert.Nil(t, err) - }) -} - -type MockExecutor struct { - ExecCommandFunc func(ctx context.Context, command []string, envs []string) (string, error) -} - -func (e *MockExecutor) ExecCommand(ctx context.Context, command []string, envs []string) (string, error) { - if e.ExecCommandFunc != nil { - return e.ExecCommandFunc(ctx, command, envs) - } - return "nil", ErrNotImplemented -} diff --git a/pkg/kb_agent/handlers/grpc_handler_test.go b/pkg/kb_agent/handlers/grpc_handler_test.go deleted file mode 100644 index f98b2f17796..00000000000 --- a/pkg/kb_agent/handlers/grpc_handler_test.go +++ /dev/null @@ -1,60 +0,0 @@ -/* -Copyright (C) 2022-2024 ApeCloud Co., Ltd - -This file is part of KubeBlocks project - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ - -package handlers - -import ( - "context" - "testing" - - "github.com/pkg/errors" - "github.com/stretchr/testify/assert" - - "github.com/apecloud/kubeblocks/pkg/kb_agent/util" -) - -func TestNewGRPCHandler(t *testing.T) { - handler, err := NewGRPCHandler(nil) - assert.NotNil(t, handler) - assert.Nil(t, err) -} - -func TestGRPCHandlerDo(t *testing.T) { - ctx := context.Background() - handler := &GRPCHandler{} - t.Run("grpc handler is nil", func(t *testing.T) { - setting := util.HandlerSpec{ - GPRC: nil, - } - do, err := handler.Do(ctx, setting, nil) - assert.Nil(t, do) - assert.NotNil(t, err) - assert.Error(t, err, errors.New("grpc setting is nil")) - }) - - t.Run("grpc handler is not nil but not implemented", func(t *testing.T) { - setting := util.HandlerSpec{ - GPRC: map[string]string{"test": "test"}, - } - do, err := handler.Do(ctx, setting, nil) - assert.Nil(t, do) - assert.NotNil(t, err) - assert.Error(t, err, ErrNotImplemented) - }) -} diff --git a/pkg/kb_agent/httpserver/apis.go b/pkg/kb_agent/httpserver/apis.go deleted file mode 100644 index 5f538bb2c08..00000000000 --- a/pkg/kb_agent/httpserver/apis.go +++ /dev/null @@ -1,123 +0,0 @@ -/* -Copyright (C) 2022-2024 ApeCloud Co., Ltd - -This file is part of KubeBlocks project - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ - -package httpserver - -import ( - "context" - "encoding/json" - "fmt" - - "github.com/pkg/errors" - "github.com/valyala/fasthttp" - - "github.com/apecloud/kubeblocks/pkg/kb_agent/handlers" - "github.com/apecloud/kubeblocks/pkg/kb_agent/util" -) - -type option = func(ctx *fasthttp.RequestCtx) - -func Endpoints() []Endpoint { - return []Endpoint{ - { - Route: util.Path, - Method: fasthttp.MethodPost, - Version: util.Version, - Handler: actionHandler, - }, - } -} - -func actionHandler(reqCtx *fasthttp.RequestCtx) { - ctx := context.Background() - body := reqCtx.PostBody() - - var req Request - if len(body) > 0 { - err := json.Unmarshal(body, &req) - if err != nil { - msg := NewErrorResponse("ERR_MALFORMED_REQUEST", fmt.Sprintf("unmarshal HTTP body failed: %v", err)) - respond(reqCtx, withError(fasthttp.StatusBadRequest, msg)) - return - } - } - - _, err := json.Marshal(req.Data) - if err != nil { - msg := NewErrorResponse("ERR_MALFORMED_REQUEST_DATA", fmt.Sprintf("marshal request data field: %v", err)) - respond(reqCtx, withError(fasthttp.StatusInternalServerError, msg)) - logger.Info("marshal request data field", "error", err.Error()) - return - } - - if req.Action == "" { - msg := NewErrorResponse("ERR_MALFORMED_REQUEST_DATA", "no action in request") - respond(reqCtx, withError(fasthttp.StatusBadRequest, msg)) - return - } - - resp, err := handlers.Do(ctx, req.Action, req.Parameters) - statusCode := fasthttp.StatusOK - if err != nil { - if errors.Is(err, handlers.ErrNotImplemented) { - statusCode = fasthttp.StatusNotImplemented - } else { - statusCode = fasthttp.StatusInternalServerError - logger.Info("action exec failed", "action", req.Action, "error", err.Error()) - } - msg := NewErrorResponse("ERR_ACTION_FAILED", fmt.Sprintf("action exec failed: %s", err.Error())) - respond(reqCtx, withError(statusCode, msg)) - return - } - - if resp == nil { - respond(reqCtx, withEmpty()) - } else { - body, _ = json.Marshal(resp) - respond(reqCtx, withJSON(statusCode, body)) - } -} - -// withJSON overrides the content-type with application/json. -func withJSON(code int, obj []byte) option { - return func(ctx *fasthttp.RequestCtx) { - ctx.Response.SetStatusCode(code) - ctx.Response.SetBody(obj) - ctx.Response.Header.SetContentType(util.JSONContentTypeHeader) - } -} - -// withError sets error code and jsonify error message. -func withError(code int, resp ErrorResponse) option { - b, _ := json.Marshal(&resp) - return withJSON(code, b) -} - -func withEmpty() option { - return func(ctx *fasthttp.RequestCtx) { - ctx.Response.SetBody(nil) - ctx.Response.SetStatusCode(fasthttp.StatusNoContent) - } -} - -func respond(ctx *fasthttp.RequestCtx, options ...option) { - for _, option := range options { - option(ctx) - } -} diff --git a/pkg/kb_agent/httpserver/apis_test.go b/pkg/kb_agent/httpserver/apis_test.go deleted file mode 100644 index e5de1f36fef..00000000000 --- a/pkg/kb_agent/httpserver/apis_test.go +++ /dev/null @@ -1,129 +0,0 @@ -/* -Copyright (C) 2022-2024 ApeCloud Co., Ltd - -This file is part of KubeBlocks project - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ - -package httpserver - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/valyala/fasthttp" - - "github.com/apecloud/kubeblocks/pkg/constant" - "github.com/apecloud/kubeblocks/pkg/kb_agent/handlers" - "github.com/apecloud/kubeblocks/pkg/kb_agent/util" - viper "github.com/apecloud/kubeblocks/pkg/viperx" -) - -func TestEndpoints(t *testing.T) { - eps := Endpoints() - assert.NotNil(t, eps) -} - -func TestActionHandler(t *testing.T) { - actionHandlerSpecs := map[string]util.HandlerSpec{ - "test": { - GPRC: map[string]string{"test": "test"}, - }, - "success": {}, - "failed": {}, - } - actionJSON, _ := json.Marshal(actionHandlerSpecs) - viper.Set(constant.KBEnvActionHandlers, string(actionJSON)) - assert.Nil(t, handlers.InitHandlers()) - - t.Run("unmarshal HTTP body failed", func(t *testing.T) { - reqCtx := &fasthttp.RequestCtx{} - reqCtx.Request.Header.SetMethod(fasthttp.MethodPost) - reqCtx.Request.Header.SetContentType("application/json") - reqCtx.Request.SetBody([]byte(`{"action":"test"`)) - actionHandler(reqCtx) - assert.Equal(t, fasthttp.StatusBadRequest, reqCtx.Response.StatusCode()) - assert.JSONEq(t, `{"errorCode":"ERR_MALFORMED_REQUEST","message":"unmarshal HTTP body failed: unexpected end of JSON input"}`, string(reqCtx.Response.Body())) - }) - - t.Run("no action in request", func(t *testing.T) { - reqCtx := &fasthttp.RequestCtx{} - reqCtx.Request.Header.SetMethod(fasthttp.MethodPost) - reqCtx.Request.Header.SetContentType("application/json") - reqCtx.Request.SetBody([]byte(`{}`)) - actionHandler(reqCtx) - assert.Equal(t, fasthttp.StatusBadRequest, reqCtx.Response.StatusCode()) - assert.JSONEq(t, `{"errorCode":"ERR_MALFORMED_REQUEST_DATA","message":"no action in request"}`, string(reqCtx.Response.Body())) - }) - - t.Run("action not implemented", func(t *testing.T) { - reqCtx := &fasthttp.RequestCtx{} - reqCtx.Request.Header.SetMethod(fasthttp.MethodPost) - reqCtx.Request.Header.SetContentType("application/json") - reqCtx.Request.SetBody([]byte(`{"action":"test"}`)) - actionHandler(reqCtx) - assert.Equal(t, fasthttp.StatusNotImplemented, reqCtx.Response.StatusCode()) - assert.JSONEq(t, `{"errorCode":"ERR_ACTION_FAILED","message":"action exec failed: not implemented"}`, string(reqCtx.Response.Body())) - }) - - t.Run("action exec failed", func(t *testing.T) { - msg := "action exec failed" - reqCtx := &fasthttp.RequestCtx{} - reqCtx.Request.Header.SetMethod(fasthttp.MethodPost) - reqCtx.Request.Header.SetContentType("application/json") - reqCtx.Request.SetBody([]byte(`{"action":"failed"}`)) - mockHandler := &MockHandler{ - DoFunc: func(ctx context.Context, setting util.HandlerSpec, args map[string]interface{}) (*handlers.Response, error) { - return nil, errors.New(msg) - }, - } - handlers.SetDefaultHandler(mockHandler) - actionHandler(reqCtx) - assert.Equal(t, fasthttp.StatusInternalServerError, reqCtx.Response.StatusCode()) - }) - - t.Run("action exec success", func(t *testing.T) { - msg := "action exec success" - reqCtx := &fasthttp.RequestCtx{} - reqCtx.Request.Header.SetMethod(fasthttp.MethodPost) - reqCtx.Request.Header.SetContentType("application/json") - reqCtx.Request.SetBody([]byte(`{"action":"success"}`)) - mockHandler := &MockHandler{ - DoFunc: func(ctx context.Context, setting util.HandlerSpec, args map[string]interface{}) (*handlers.Response, error) { - return &handlers.Response{Message: msg}, nil - }, - } - handlers.SetDefaultHandler(mockHandler) - actionHandler(reqCtx) - assert.Equal(t, fasthttp.StatusOK, reqCtx.Response.StatusCode()) - expectedResponse := fmt.Sprintf(`{"message":"%s"}`, msg) - assert.Equal(t, expectedResponse, string(reqCtx.Response.Body())) - }) -} - -type MockHandler struct { - DoFunc func(ctx context.Context, setting util.HandlerSpec, args map[string]interface{}) (*handlers.Response, error) -} - -func (h *MockHandler) Do(ctx context.Context, setting util.HandlerSpec, args map[string]interface{}) (*handlers.Response, error) { - if h.DoFunc != nil { - return h.DoFunc(ctx, setting, args) - } - return nil, handlers.ErrNotImplemented -} diff --git a/pkg/kb_agent/httpserver/config.go b/pkg/kb_agent/httpserver/config.go deleted file mode 100644 index c83bb685445..00000000000 --- a/pkg/kb_agent/httpserver/config.go +++ /dev/null @@ -1,50 +0,0 @@ -/* -Copyright (C) 2022-2024 ApeCloud Co., Ltd - -This file is part of KubeBlocks project - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ - -package httpserver - -import ( - "fmt" - - "github.com/spf13/pflag" - ctrl "sigs.k8s.io/controller-runtime" -) - -const KBAgentDefaultPort = 3501 -const KBAgentDefaultConcurrency = 10 - -type Config struct { - Port int - Address string - ConCurrency int - UnixDomainSocket string - APILogging bool -} - -var config Config -var logger = ctrl.Log.WithName("HTTPServer") - -func init() { - pflag.IntVar(&config.Port, "port", KBAgentDefaultPort, "The HTTP Server listen port for kb-agent service.") - pflag.IntVar(&config.ConCurrency, "max-concurrency", KBAgentDefaultConcurrency, - fmt.Sprintf("The maximum number of concurrent connections the Server may serve, use the default value %d if <=0.", KBAgentDefaultConcurrency)) - pflag.StringVar(&config.Address, "address", "0.0.0.0", "The HTTP Server listen address for kb-agent service.") - pflag.StringVar(&config.UnixDomainSocket, "unix-socket", ".", "The path of the Unix Domain Socket for kb-agent service.") - pflag.BoolVar(&config.APILogging, "api-logging", true, "Enable api logging for kb-agent request.") -} diff --git a/pkg/kb_agent/httpserver/endpoint.go b/pkg/kb_agent/httpserver/endpoint.go deleted file mode 100644 index c80b6e7650c..00000000000 --- a/pkg/kb_agent/httpserver/endpoint.go +++ /dev/null @@ -1,47 +0,0 @@ -/* -Copyright (C) 2022-2024 ApeCloud Co., Ltd - -This file is part of KubeBlocks project - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ - -package httpserver - -import ( - "github.com/valyala/fasthttp" -) - -type Endpoint struct { - Method string - Route string - - // Version represents the version of the API, which is currently v1.0. - // the version is introduced to allow breaking changes. - // If the API is upgraded to v2.0, the v1.0 API will be maintained - // for compatibility until all legacy accesses are removed. - Version string - - // LegacyRoute is used When the API is upgraded, some old APIs may - // need to update their path routes. To ensure compatibility, - // the old paths can be set as legacyRoute. - LegacyRoute string - Handler fasthttp.RequestHandler -} - -type Request struct { - Action string `json:"action"` - Data interface{} `json:"data,omitempty"` - Parameters map[string]any `json:"parameters,omitempty"` -} diff --git a/pkg/kb_agent/httpserver/server.go b/pkg/kb_agent/httpserver/server.go deleted file mode 100644 index 3f975a76e59..00000000000 --- a/pkg/kb_agent/httpserver/server.go +++ /dev/null @@ -1,161 +0,0 @@ -/* -Copyright (C) 2022-2024 ApeCloud Co., Ltd - -This file is part of KubeBlocks project - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ - -package httpserver - -import ( - "errors" - "fmt" - "io" - "net" - "time" - - fasthttprouter "github.com/fasthttp/router" - "github.com/valyala/fasthttp" -) - -// Server is an interface for the kb-agent HTTP server. -type Server interface { - io.Closer - Router() fasthttp.RequestHandler - StartNonBlocking() error -} - -type server struct { - config Config - endpoints []Endpoint - servers []*fasthttp.Server -} - -// NewServer returns a new HTTP server. -func NewServer() Server { - return &server{ - endpoints: Endpoints(), - config: config, - } -} - -// StartNonBlocking starts a new server in a goroutine. -func (s *server) StartNonBlocking() error { - logger.Info("Starting HTTP Server") - handler := s.Router() - - APILogging := s.config.APILogging - if APILogging { - handler = s.apiLogger(handler) - } - - var listeners []net.Listener - if s.config.UnixDomainSocket != "" { - socket := fmt.Sprintf("%s/kb_agent.socket", s.config.UnixDomainSocket) - l, err := net.Listen("unix", socket) - if err != nil { - return err - } - listeners = append(listeners, l) - } else { - apiListenAddress := s.config.Address - l, err := net.Listen("tcp", fmt.Sprintf("%s:%v", apiListenAddress, s.config.Port)) - if err != nil { - logger.Error(err, "listen address", apiListenAddress, "port", s.config.Port) - } else { - listeners = append(listeners, l) - } - } - - if len(listeners) == 0 { - return errors.New("no endpoint to listen on") - } - - for _, listener := range listeners { - // customServer is created in a loop because each instance - // has a handle on the underlying listener. - customServer := &fasthttp.Server{ - Handler: handler, - } - - if s.config.ConCurrency > 0 { - customServer.Concurrency = s.config.ConCurrency - } else { - customServer.Concurrency = KBAgentDefaultConcurrency - } - - s.servers = append(s.servers, customServer) - go func(l net.Listener) { - if err := customServer.Serve(l); err != nil { - panic(err) - } - }(listener) - } - - return nil -} - -func (s *server) apiLogger(next fasthttp.RequestHandler) fasthttp.RequestHandler { - return func(ctx *fasthttp.RequestCtx) { - reqLogger := logger - if userAgent := string(ctx.Request.Header.Peek("User-Agent")); userAgent != "" { - reqLogger = logger.WithValues("useragent", userAgent) - } - start := time.Now() - path := string(ctx.Path()) - reqLogger.Info("HTTP API Called", "method", string(ctx.Method()), "path", path) - next(ctx) - elapsed := float64(time.Since(start) / time.Millisecond) - reqLogger.Info("HTTP API Called", "status code", ctx.Response.StatusCode(), "cost", elapsed) - } -} - -func (s *server) Router() fasthttp.RequestHandler { - router := s.getRouter(s.endpoints) - - return router.Handler -} - -func (s *server) getRouter(endpoints []Endpoint) *fasthttprouter.Router { - router := fasthttprouter.New() - for _, e := range endpoints { - path := fmt.Sprintf("/%s/%s", e.Version, e.Route) - router.Handle(e.Method, path, e.Handler) - - if e.LegacyRoute != "" { - path := fmt.Sprintf("/%s/%s", e.Version, e.LegacyRoute) - router.Handle(e.Method, path, e.Handler) - } - } - for method, path := range router.List() { - logger.Info("API route path", "method", method, "path", path) - } - - return router -} - -func (s *server) Close() error { - errs := make([]error, len(s.servers)) - - for i, ln := range s.servers { - // This calls `Close()` on the underlying listener. - if err := ln.Shutdown(); err != nil { - logger.Error(err, "server close failed") - errs[i] = err - } - } - - return errors.Join() -} diff --git a/pkg/kb_agent/httpserver/server_test.go b/pkg/kb_agent/httpserver/server_test.go deleted file mode 100644 index 698a6d7ec4d..00000000000 --- a/pkg/kb_agent/httpserver/server_test.go +++ /dev/null @@ -1,100 +0,0 @@ -/* -Copyright (C) 2022-2024 ApeCloud Co., Ltd - -This file is part of KubeBlocks project - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ - -package httpserver - -import ( - "os" - "testing" - - "github.com/pkg/errors" - "github.com/stretchr/testify/assert" - "github.com/valyala/fasthttp" - - "github.com/apecloud/kubeblocks/pkg/kb_agent/util" -) - -func TestNewServer(t *testing.T) { - s := NewServer() - assert.NotNil(t, s) -} - -func TestStartNonBlocking(t *testing.T) { - - t.Run("StartNonBlocking unix domain socket", func(t *testing.T) { - config := Config{ - Port: KBAgentDefaultPort, - Address: "0.0.0.0", - ConCurrency: 0, - UnixDomainSocket: ".", - APILogging: true, - } - s := &server{ - config: config, - endpoints: Endpoints(), - } - _ = os.Remove(config.UnixDomainSocket + "/kb_agent.socket") - err := s.StartNonBlocking() - assert.Nil(t, err) - err = s.Close() - assert.Nil(t, err) - }) - - t.Run("StartNonBlocking tcp socket", func(t *testing.T) { - config := Config{ - Port: KBAgentDefaultPort, - Address: "0.0.0.0", - ConCurrency: KBAgentDefaultConcurrency, - UnixDomainSocket: "", - APILogging: true, - } - s := &server{ - config: config, - endpoints: []Endpoint{ - { - Route: util.Path, - Method: fasthttp.MethodPost, - Version: util.Version, - Handler: actionHandler, - LegacyRoute: "test_legacy_route", - }, - }, - } - err := s.StartNonBlocking() - assert.Nil(t, err) - err = s.Close() - assert.Nil(t, err) - }) - - t.Run("StartNonBlocking zero listeners", func(t *testing.T) { - config := Config{ - Port: -1, - Address: "0.0.0.0", - ConCurrency: KBAgentDefaultConcurrency, - UnixDomainSocket: "", - APILogging: false, - } - s := &server{ - config: config, - endpoints: Endpoints(), - } - err := s.StartNonBlocking() - assert.Error(t, err, errors.New("no endpoint to listen on")) - }) -} diff --git a/pkg/kb_agent/util/command.go b/pkg/kb_agent/util/command.go deleted file mode 100644 index c39fddbc3d2..00000000000 --- a/pkg/kb_agent/util/command.go +++ /dev/null @@ -1,62 +0,0 @@ -/* -Copyright (C) 2022-2024 ApeCloud Co., Ltd - -This file is part of KubeBlocks project - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ - -package util - -import ( - "context" - "fmt" - "os" - "os/exec" - "strings" - - "github.com/pkg/errors" -) - -type Executor interface { - ExecCommand(ctx context.Context, command []string, envs []string) (string, error) -} - -type ExecutorImpl struct{} - -func (e *ExecutorImpl) ExecCommand(ctx context.Context, command []string, envs []string) (string, error) { - return ExecCommand(ctx, command, envs) -} - -func ExecCommand(ctx context.Context, command []string, envs []string) (string, error) { - if len(command) == 0 { - return "", errors.New("command can not be empty") - } - cmd := exec.CommandContext(ctx, command[0], command[1:]...) - cmd.Env = envs - bytes, err := cmd.Output() - if exitErr, ok := err.(*exec.ExitError); ok { - err = errors.New(string(exitErr.Stderr)) - } - return string(bytes), err -} - -func GetAllEnvs(args map[string]any) []string { - envs := os.Environ() - for k, v := range args { - env := fmt.Sprintf("%s=%v", strings.ToUpper(k), v) - envs = append(envs, env) - } - return envs -} diff --git a/pkg/kb_agent/util/event_test.go b/pkg/kb_agent/util/event_test.go deleted file mode 100644 index 7d04613a001..00000000000 --- a/pkg/kb_agent/util/event_test.go +++ /dev/null @@ -1,63 +0,0 @@ -/* -Copyright (C) 2022-2024 ApeCloud Co., Ltd - -This file is part of KubeBlocks project - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ - -package util - -import ( - "context" - "errors" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestSentEventForProbe(t *testing.T) { - ctx := context.Background() - t.Run("action is empty", func(t *testing.T) { - msg := &MockNilActionMessage{} - err := SentEventForProbe(ctx, msg) - assert.Error(t, errors.New("action is unset"), err) - }) - t.Run("action is not empty", func(t *testing.T) { - msg := &MockNotNilActionMessage{} - err := SentEventForProbe(ctx, msg) - assert.Nil(t, err) - }) -} - -func TestCreateEvent(t *testing.T) { - msg := &MockNotNilActionMessage{} - event, err := CreateEvent("test", msg) - assert.Nil(t, err) - assert.NotEqual(t, nil, event) -} - -type MockNotNilActionMessage struct { -} - -func (m *MockNotNilActionMessage) GetAction() string { - return "test" -} - -type MockNilActionMessage struct { -} - -func (m *MockNilActionMessage) GetAction() string { - return "" -} diff --git a/pkg/kb_agent/util/types.go b/pkg/kb_agent/util/types.go deleted file mode 100644 index 0a82209bcfc..00000000000 --- a/pkg/kb_agent/util/types.go +++ /dev/null @@ -1,66 +0,0 @@ -/* -Copyright (C) 2022-2024 ApeCloud Co., Ltd - -This file is part of KubeBlocks project - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ - -package util - -const ( - OperationSuccess = "Success" - OperationFailed = "Failed" - KBAgentEventFieldPath = "spec.containers{kb-agent}" -) - -// for http server -const ( - JSONContentTypeHeader = "application/json" - Version = "v1.0" - Path = "/action" -) - -type CronJob struct { - PeriodSeconds int `json:"periodSeconds,omitempty"` - SuccessThreshold int `json:"successThreshold,omitempty"` - FailureThreshold int `json:"failureThreshold,omitempty"` - ReportFrequency int `json:"reportFrequency,omitempty"` -} - -type HandlerSpec struct { - TimeoutSeconds int `json:"timeoutSeconds,omitempty"` - Command []string `json:"command,omitempty"` - GPRC map[string]string `json:"grpc,omitempty"` - CronJob *CronJob `json:"cronJob,omitempty"` -} - -type ActionMessage interface { - GetAction() string -} - -type MessageBase struct { - Event string `json:"event,omitempty"` - Message string `json:"message,omitempty"` - Action string `json:"action,omitempty"` -} - -func (m MessageBase) GetAction() string { - return m.Action -} - -type RoleProbeMessage struct { - MessageBase - Role string `json:"role,omitempty"` -} diff --git a/pkg/kbagent/client/client.go b/pkg/kbagent/client/client.go new file mode 100644 index 00000000000..d26e12a999c --- /dev/null +++ b/pkg/kbagent/client/client.go @@ -0,0 +1,98 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package client + +import ( + "context" + "fmt" + "net" + "net/http" + "time" + + corev1 "k8s.io/api/core/v1" + + intctrlutil "github.com/apecloud/kubeblocks/pkg/controllerutil" + "github.com/apecloud/kubeblocks/pkg/kbagent/proto" +) + +// TODO: move to a common package +const ( + kbAgentContainerName = "kbagent" + kbAgentPortName = "http" +) + +type Client interface { + CallAction(ctx context.Context, req proto.ActionRequest) (proto.ActionResponse, error) + + // LaunchProbe(ctx context.Context, probe proto.Probe) error +} + +// HACK: for unit test only. +var mockClient Client +var mockClientError error + +func SetMockClient(cli Client, err error) { + mockClient = cli + mockClientError = err +} + +func UnsetMockClient() { + mockClient = nil + mockClientError = nil +} + +func GetMockClient() Client { + return mockClient +} + +func NewClient(pod corev1.Pod) (Client, error) { + if mockClient != nil || mockClientError != nil { + return mockClient, mockClientError + } + + port, err := intctrlutil.GetPortByName(pod, kbAgentContainerName, kbAgentPortName) + if err != nil { + // has no kb-agent defined + return nil, nil + } + + ip := pod.Status.PodIP + if ip == "" { + return nil, fmt.Errorf("pod %v has no ip", pod.Name) + } + + // don't use default http-client + dialer := &net.Dialer{ + Timeout: 5 * time.Second, + } + transport := &http.Transport{ + Dial: dialer.Dial, + TLSHandshakeTimeout: 5 * time.Second, + } + cli := &http.Client{ + Timeout: time.Second * 30, + Transport: transport, + } + return &httpClient{ + host: ip, + port: port, + client: cli, + }, nil +} diff --git a/pkg/kbagent/client/client_mock.go b/pkg/kbagent/client/client_mock.go new file mode 100644 index 00000000000..ef37a8d9b92 --- /dev/null +++ b/pkg/kbagent/client/client_mock.go @@ -0,0 +1,72 @@ +// /* +// Copyright (C) 2022-2024 ApeCloud Co., Ltd +// +// This file is part of KubeBlocks project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . +// */ +// +// + +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/apecloud/kubeblocks/pkg/kbagent/client (interfaces: Client) + +// Package client is a generated GoMock package. +package client + +import ( + context "context" + reflect "reflect" + + proto "github.com/apecloud/kubeblocks/pkg/kbagent/proto" + gomock "github.com/golang/mock/gomock" +) + +// MockClient is a mock of Client interface. +type MockClient struct { + ctrl *gomock.Controller + recorder *MockClientMockRecorder +} + +// MockClientMockRecorder is the mock recorder for MockClient. +type MockClientMockRecorder struct { + mock *MockClient +} + +// NewMockClient creates a new mock instance. +func NewMockClient(ctrl *gomock.Controller) *MockClient { + mock := &MockClient{ctrl: ctrl} + mock.recorder = &MockClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockClient) EXPECT() *MockClientMockRecorder { + return m.recorder +} + +// CallAction mocks base method. +func (m *MockClient) CallAction(arg0 context.Context, arg1 proto.ActionRequest) (proto.ActionResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CallAction", arg0, arg1) + ret0, _ := ret[0].(proto.ActionResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CallAction indicates an expected call of CallAction. +func (mr *MockClientMockRecorder) CallAction(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CallAction", reflect.TypeOf((*MockClient)(nil).CallAction), arg0, arg1) +} diff --git a/pkg/kb_agent/handlers/interface.go b/pkg/kbagent/client/generate.go similarity index 71% rename from pkg/kb_agent/handlers/interface.go rename to pkg/kbagent/client/generate.go index 1e411556b56..0d6537069d2 100644 --- a/pkg/kb_agent/handlers/interface.go +++ b/pkg/kbagent/client/generate.go @@ -17,19 +17,6 @@ You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ -package handlers +package client -import ( - "context" - - "github.com/apecloud/kubeblocks/pkg/kb_agent/util" -) - -type Response struct { - Message string `json:"message"` -} - -type Handler interface { - // exec action - Do(ctx context.Context, settings util.HandlerSpec, agrs map[string]interface{}) (*Response, error) -} +//go:generate go run github.com/golang/mock/mockgen -copyright_file ../../../hack/boilerplate.go.txt -package client -destination client_mock.go github.com/apecloud/kubeblocks/pkg/kbagent/client Client diff --git a/pkg/kbagent/client/httpclient.go b/pkg/kbagent/client/httpclient.go new file mode 100644 index 00000000000..905738063c8 --- /dev/null +++ b/pkg/kbagent/client/httpclient.go @@ -0,0 +1,105 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package client + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + + "github.com/valyala/fasthttp" + + "github.com/apecloud/kubeblocks/pkg/kbagent/proto" +) + +const ( + urlTemplate = "http://%s:%d/%s" + actionServiceURI = "/v1.0/action" +) + +type httpClient struct { + host string + port int32 + client *http.Client +} + +var _ Client = &httpClient{} + +func (c *httpClient) CallAction(ctx context.Context, req proto.ActionRequest) (proto.ActionResponse, error) { + url := fmt.Sprintf(urlTemplate, c.host, c.port, actionServiceURI) + + data, err := json.Marshal(req) + if err != nil { + return proto.ActionResponse{}, err + } + + payload, err := c.request(ctx, fasthttp.MethodPost, url, bytes.NewReader(data)) + if err != nil { + return proto.ActionResponse{}, err + } + if payload == nil { + return proto.ActionResponse{}, nil + } + return c.decode(payload) +} + +func (c *httpClient) request(ctx context.Context, method, url string, body io.Reader) (io.Reader, error) { + req, err := http.NewRequestWithContext(ctx, method, url, body) + if err != nil { + return nil, err + } + + rsp, err := c.client.Do(req) + if err != nil { + return nil, err + } + defer rsp.Body.Close() + + switch rsp.StatusCode { + case http.StatusOK, http.StatusUnavailableForLegalReasons: + return rsp.Body, nil + case http.StatusNoContent: + return nil, nil + case http.StatusNotImplemented, http.StatusInternalServerError: + fallthrough + default: + msg, err := io.ReadAll(rsp.Body) + if err != nil { + return nil, err + } + return nil, fmt.Errorf("%s", string(msg)) + } +} + +func (c *httpClient) decode(body io.Reader) (proto.ActionResponse, error) { + rsp := proto.ActionResponse{} + data, err := io.ReadAll(body) + if err != nil { + return rsp, err + } + err = json.Unmarshal(data, &rsp) + if err != nil { + return rsp, err + } + return rsp, nil +} diff --git a/pkg/kbagent/proto/proto.go b/pkg/kbagent/proto/proto.go new file mode 100644 index 00000000000..c8b70141fbd --- /dev/null +++ b/pkg/kbagent/proto/proto.go @@ -0,0 +1,68 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package proto + +import "time" + +type Action struct { + Name string `json:"name"` + Exec *ExecAction `json:"exec,omitempty"` + TimeoutSeconds int32 `json:"timeoutSeconds,omitempty"` + RetryPolicy *RetryPolicy `json:"retryPolicy,omitempty"` +} + +type ExecAction struct { + Commands []string `json:"command,omitempty"` + Args []string `json:"args,omitempty"` + Env []string `json:"env,omitempty"` +} + +type RetryPolicy struct { + MaxRetries int `json:"maxRetries,omitempty"` + RetryInterval time.Duration `json:"retryInterval,omitempty"` +} + +type ActionRequest struct { + Action string `json:"action"` + Parameters map[string]string `json:"parameters,omitempty"` + NonBlocking *bool `json:"nonBlocking,omitempty"` + TimeoutSeconds *int32 `json:"timeoutSeconds,omitempty"` + RetryPolicy *RetryPolicy `json:"retryPolicy,omitempty"` +} + +type ActionResponse struct { + Output []byte `json:"output,omitempty"` +} + +type Probe struct { + Action string `json:"action"` + InitialDelaySeconds int32 `json:"initialDelaySeconds,omitempty"` + PeriodSeconds int32 `json:"periodSeconds,omitempty"` + SuccessThreshold int32 `json:"successThreshold,omitempty"` + FailureThreshold int32 `json:"failureThreshold,omitempty"` + ReportPeriodSeconds *int32 `json:"reportPeriodSeconds,omitempty"` +} + +type ProbeEvent struct { + Probe string `json:"probe,omitempty"` + Code int32 `json:"code,omitempty"` + Output []byte `json:"output,omitempty"` + Message string `json:"message,omitempty"` +} diff --git a/pkg/kbagent/server/httpserver.go b/pkg/kbagent/server/httpserver.go new file mode 100644 index 00000000000..fc1342d3e83 --- /dev/null +++ b/pkg/kbagent/server/httpserver.go @@ -0,0 +1,235 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package server + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "strings" + "time" + + fasthttprouter "github.com/fasthttp/router" + "github.com/go-logr/logr" + "github.com/valyala/fasthttp" + + "github.com/apecloud/kubeblocks/pkg/kbagent/service" +) + +const ( + defaultMaxConcurrency = 8 + jsonContentTypeHeader = "application/json" +) + +type server struct { + logger logr.Logger + config Config + services []service.Service + servers []*fasthttp.Server +} + +var _ Server = &server{} + +// StartNonBlocking starts a new server in a goroutine. +func (s *server) StartNonBlocking() error { + s.logger.Info("starting HTTP server") + + // start all services first + for i := range s.services { + if err := s.services[i].Start(); err != nil { + s.logger.Error(err, fmt.Sprintf("start service %s failed", s.services[i].Kind())) + return err + } + s.logger.Info(fmt.Sprintf("service %s started...", s.services[i].Kind())) + } + + handler := s.router() + if s.config.Logging { + handler = s.apiLogger(handler) + } + + var listeners []net.Listener + if s.config.UnixDomainSocket != "" { + socket := fmt.Sprintf("%s/kbagent.socket", s.config.UnixDomainSocket) + l, err := net.Listen("unix", socket) + if err != nil { + return err + } + listeners = append(listeners, l) + } else { + l, err := net.Listen("tcp", fmt.Sprintf("%s:%v", s.config.Address, s.config.Port)) + if err != nil { + s.logger.Error(err, "listen address", s.config.Address, "port", s.config.Port) + } else { + listeners = append(listeners, l) + } + } + + if len(listeners) == 0 { + return errors.New("no endpoint to listen on") + } + + for _, listener := range listeners { + // customServer is created in a loop because each instance + // has a handle on the underlying listener. + customServer := &fasthttp.Server{ + Handler: handler, + } + + if s.config.Concurrency > 0 { + customServer.Concurrency = s.config.Concurrency + } else { + customServer.Concurrency = defaultMaxConcurrency + } + + s.servers = append(s.servers, customServer) + go func(l net.Listener) { + if err := customServer.Serve(l); err != nil { + panic(err) + } + }(listener) + } + + return nil +} + +func (s *server) Close() error { + errs := make([]error, len(s.servers)) + + for i, ln := range s.servers { + // This calls `Close()` on the underlying listener. + if err := ln.Shutdown(); err != nil { + s.logger.Error(err, "server close failed") + errs[i] = err + } + } + + return errors.Join() +} + +func (s *server) apiLogger(next fasthttp.RequestHandler) fasthttp.RequestHandler { + return func(ctx *fasthttp.RequestCtx) { + reqLogger := s.logger + if userAgent := string(ctx.Request.Header.Peek("User-Agent")); userAgent != "" { + reqLogger = s.logger.WithValues("useragent", userAgent) + } + start := time.Now() + path := string(ctx.Path()) + reqLogger.Info("HTTP API Called", "method", string(ctx.Method()), "path", path) + next(ctx) + elapsed := float64(time.Since(start) / time.Millisecond) + reqLogger.Info("HTTP API Called", "status code", ctx.Response.StatusCode(), "cost", elapsed) + } +} + +func (s *server) router() fasthttp.RequestHandler { + router := fasthttprouter.New() + for i := range s.services { + s.registerService(router, s.services[i]) + } + return router.Handler +} + +func (s *server) registerService(router *fasthttprouter.Router, svc service.Service) { + router.Handle(fasthttp.MethodPost, s.serviceURI(svc), s.dispatcher(svc)) + s.logger.Info("register service to server", "service", svc.Kind(), "method", fasthttp.MethodPost, "uri", s.serviceURI(svc)) +} + +func (s *server) serviceURI(svc service.Service) string { + return fmt.Sprintf("/%s/%s", svc.Version(), strings.ToLower(svc.Kind())) +} + +func (s *server) dispatcher(svc service.Service) func(*fasthttp.RequestCtx) { + return func(reqCtx *fasthttp.RequestCtx) { + ctx := context.Background() + body := reqCtx.PostBody() + + req, err := svc.Decode(body) + if err != nil { + msg := newErrorResponse("ERR_MALFORMED_REQUEST", fmt.Sprintf("unmarshal HTTP body failed: %v", err)) + respond(reqCtx, withError(fasthttp.StatusBadRequest, msg)) + return + } + + rsp, err := svc.HandleRequest(ctx, req) + statusCode := fasthttp.StatusOK + if err != nil { + if errors.Is(err, service.ErrNotImplemented) { + statusCode = fasthttp.StatusNotImplemented + } else { + statusCode = fasthttp.StatusInternalServerError + } + + s.logger.Info("service call failed", "service", svc.Kind(), "error", err.Error()) + + msg := newErrorResponse("ERR_SERVICE_FAILED", fmt.Sprintf("service call failed: %s", err.Error())) + respond(reqCtx, withError(statusCode, msg)) + return + } + + if rsp == nil { + respond(reqCtx, withEmpty()) + } else { + respond(reqCtx, withJSON(statusCode, rsp)) + } + } +} + +type errorResponse struct { + ErrorCode string `json:"errorCode"` + Message string `json:"message"` +} + +func newErrorResponse(errorCode, message string) errorResponse { + return errorResponse{ + ErrorCode: errorCode, + Message: message, + } +} + +type option = func(ctx *fasthttp.RequestCtx) + +func withJSON(code int, obj []byte) option { + return func(ctx *fasthttp.RequestCtx) { + ctx.Response.SetStatusCode(code) + ctx.Response.SetBody(obj) + ctx.Response.Header.SetContentType(jsonContentTypeHeader) + } +} + +func withError(code int, resp errorResponse) option { + b, _ := json.Marshal(&resp) + return withJSON(code, b) +} + +func withEmpty() option { + return func(ctx *fasthttp.RequestCtx) { + ctx.Response.SetBody(nil) + ctx.Response.SetStatusCode(fasthttp.StatusNoContent) + } +} + +func respond(ctx *fasthttp.RequestCtx, options ...option) { + for _, option := range options { + option(ctx) + } +} diff --git a/pkg/kbagent/server/types.go b/pkg/kbagent/server/types.go new file mode 100644 index 00000000000..193dd964cfa --- /dev/null +++ b/pkg/kbagent/server/types.go @@ -0,0 +1,51 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package server + +import ( + "io" + + "github.com/go-logr/logr" + + "github.com/apecloud/kubeblocks/pkg/kbagent/service" +) + +// Server is an interface for the kb-agent server. +type Server interface { + io.Closer + StartNonBlocking() error +} + +type Config struct { + Address string + UnixDomainSocket string + Port int + Concurrency int + Logging bool +} + +// NewHTTPServer returns a new HTTP server. +func NewHTTPServer(logger logr.Logger, config Config, services []service.Service) Server { + return &server{ + logger: logger, + config: config, + services: services, + } +} diff --git a/pkg/kbagent/service/action.go b/pkg/kbagent/service/action.go new file mode 100644 index 00000000000..7eea194726a --- /dev/null +++ b/pkg/kbagent/service/action.go @@ -0,0 +1,132 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package service + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/go-logr/logr" + "github.com/pkg/errors" + "golang.org/x/exp/maps" + + "github.com/apecloud/kubeblocks/pkg/kbagent/proto" +) + +const ( + actionServiceName = "Action" + actionServiceVersion = "v1.0" +) + +func newActionService(logger logr.Logger, actions []proto.Action) (*actionService, error) { + sa := &actionService{ + logger: logger, + actions: make(map[string]*proto.Action), + runningActions: map[string]*runningAction{}, + } + for i, action := range actions { + sa.actions[action.Name] = &actions[i] + } + logger.Info(fmt.Sprintf("create service %s", sa.Kind()), "actions", strings.Join(maps.Keys(sa.actions), ",")) + return sa, nil +} + +type actionService struct { + logger logr.Logger + actions map[string]*proto.Action + runningActions map[string]*runningAction +} + +type runningAction struct { + stdoutChan chan []byte + stderrChan chan []byte + errChan chan error +} + +var _ Service = &actionService{} + +func (s *actionService) Kind() string { + return actionServiceName +} + +func (s *actionService) Version() string { + return actionServiceVersion +} + +func (s *actionService) Start() error { + return nil +} + +func (s *actionService) Decode(payload []byte) (interface{}, error) { + req := &proto.ActionRequest{} + if err := json.Unmarshal(payload, req); err != nil { + return nil, err + } + return req, nil +} + +func (s *actionService) HandleRequest(ctx context.Context, i interface{}) ([]byte, error) { + req := i.(*proto.ActionRequest) + if _, ok := s.actions[req.Action]; !ok { + return nil, errors.Wrapf(ErrNotDefined, "%s is not defined", req.Action) + } + return s.handleActionRequest(ctx, req) +} + +func (s *actionService) handleActionRequest(ctx context.Context, req *proto.ActionRequest) ([]byte, error) { + action := s.actions[req.Action] + if action.Exec != nil { + return s.handleExecAction(ctx, req, action) + } + return nil, errors.Wrap(ErrNotImplemented, "only exec action is supported") +} + +func (s *actionService) handleExecAction(ctx context.Context, req *proto.ActionRequest, action *proto.Action) ([]byte, error) { + if req.NonBlocking != nil && *req.NonBlocking { + return s.handleExecActionNonBlocking(ctx, req, action) + } + return runCommand(ctx, action.Exec, req.Parameters, req.TimeoutSeconds) +} + +func (s *actionService) handleExecActionNonBlocking(ctx context.Context, req *proto.ActionRequest, action *proto.Action) ([]byte, error) { + running, ok := s.runningActions[req.Action] + if !ok { + stdoutChan, stderrChan, errChan, err := runCommandNonBlocking(ctx, action.Exec, req.Parameters, req.TimeoutSeconds) + if err != nil { + return nil, err + } + running = &runningAction{ + stdoutChan: stdoutChan, + stderrChan: stderrChan, + errChan: errChan, + } + s.runningActions[req.Action] = running + } + err := gather(running.errChan) + if err == nil { + return nil, ErrInProgress + } + if *err != nil { + return nil, *err + } + return *gather(running.stdoutChan), nil +} diff --git a/pkg/kbagent/service/command.go b/pkg/kbagent/service/command.go new file mode 100644 index 00000000000..04f799903e7 --- /dev/null +++ b/pkg/kbagent/service/command.go @@ -0,0 +1,212 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package service + +import ( + "bytes" + "context" + "io" + "os" + "os/exec" + "sync" + "time" + + "github.com/pkg/errors" + + "github.com/apecloud/kubeblocks/pkg/kbagent/proto" + "github.com/apecloud/kubeblocks/pkg/kbagent/util" +) + +const ( + defaultBufferSize = 4096 +) + +func gather[T interface{}](ch chan T) *T { + select { + case v := <-ch: + return &v + default: + return nil + } +} + +func runCommand(ctx context.Context, action *proto.ExecAction, parameters map[string]string, timeout *int32) ([]byte, error) { + stdoutChan, _, errChan, err := runCommandNonBlocking(ctx, action, parameters, timeout) + if err != nil { + return nil, err + } + err = <-errChan + if err != nil { + return nil, err + } + return <-stdoutChan, nil +} + +func runCommandNonBlocking(ctx context.Context, action *proto.ExecAction, parameters map[string]string, timeout *int32) (chan []byte, chan []byte, chan error, error) { + stdoutBuf := bytes.NewBuffer(make([]byte, 0, defaultBufferSize)) + stderrBuf := bytes.NewBuffer(make([]byte, 0, defaultBufferSize)) + execErrorChan, err := runCommandX(ctx, action, parameters, timeout, nil, stdoutBuf, stderrBuf) + if err != nil { + return nil, nil, nil, err + } + + stdoutChan := make(chan []byte, defaultBufferSize) + stderrChan := make(chan []byte, defaultBufferSize) + errChan := make(chan error) + go func() { + defer close(errChan) + defer close(stderrChan) + defer close(stdoutChan) + + // wait for the command to finish + execErr, ok := <-execErrorChan + if !ok { + execErr = errors.New("runtime error: error chan closed unexpectedly") + } + + stdoutChan <- stdoutBuf.Bytes() + stderrChan <- stderrBuf.Bytes() + errChan <- execErr + }() + return stdoutChan, stderrChan, errChan, nil +} + +func runCommandX(ctx context.Context, action *proto.ExecAction, parameters map[string]string, timeout *int32, + stdinReader io.Reader, stdoutWriter, stderrWriter io.Writer) (chan error, error) { + if timeout != nil && *timeout > 0 { + timeoutCtx, cancel := context.WithTimeout(ctx, time.Duration(*timeout)*time.Second) + defer cancel() + ctx = timeoutCtx + } + + mergedArgs := func() []string { + args := make([]string, 0) + if len(action.Commands) > 1 { + args = append(args, action.Commands[1:]...) + } + args = append(args, action.Args...) + return args + }() + + mergedEnv := func() []string { + // env order: parameters (action specific variables) | os env (defined by vars) | user-defined env in action + env := util.EnvM2L(parameters) + if len(env) > 0 || len(action.Env) > 0 { + env = append(env, os.Environ()...) + } + if len(action.Env) > 0 { + env = append(env, action.Env...) + } + return env + }() + + cmd := exec.CommandContext(ctx, action.Commands[0], mergedArgs...) + if len(mergedEnv) > 0 { + cmd.Env = mergedEnv + } + + var ( + stdin io.WriteCloser + stdout, stderr io.ReadCloser + ) + if stdinReader != nil { + var stdinErr error + stdin, stdinErr = cmd.StdinPipe() + if stdinErr != nil { + return nil, errors.Wrapf(ErrInternalError, "failed to create stdin pipe: %v", stdinErr) + } + } + if stdoutWriter != nil { + var stdoutErr error + stdout, stdoutErr = cmd.StdoutPipe() + if stdoutErr != nil { + return nil, errors.Wrapf(ErrInternalError, "failed to create stdout pipe: %v", stdoutErr) + } + } + if stderrWriter != nil { + var stderrErr error + stderr, stderrErr = cmd.StderrPipe() + if stderrErr != nil { + return nil, errors.Wrapf(ErrInternalError, "failed to create stderr pipe: %v", stderrErr) + } + } + + errChan := make(chan error) + go func() { + defer close(errChan) + + if err := cmd.Start(); err != nil { + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + errChan <- ErrTimeout + } else { + errChan <- errors.Wrapf(ErrFailed, "failed to start command: %v", err) + } + return + } + + var wg sync.WaitGroup + wg.Add(3) + + go func() { + defer wg.Done() + if stdinReader != nil { + defer stdin.Close() + _, copyErr := io.Copy(stdin, stdinReader) + if copyErr != nil { + errChan <- errors.Wrapf(ErrFailed, "failed to copy from input reader to stdin: %v", copyErr) + } + } + }() + go func() { + defer wg.Done() + if stdoutWriter != nil { + _, copyErr := io.Copy(stdoutWriter, stdout) + if copyErr != nil { + errChan <- errors.Wrapf(ErrFailed, "failed to copy stdout to output writer: %v", copyErr) + } + } + }() + go func() { + defer wg.Done() + if stderrWriter != nil { + _, copyErr := io.Copy(stderrWriter, stderr) + if copyErr != nil { + errChan <- errors.Wrapf(ErrFailed, "failed to copy stderr to error writer: %v", copyErr) + } + } + }() + + wg.Wait() + + execErr := cmd.Wait() + if execErr != nil { + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + execErr = ErrTimeout + } else { + var exitErr *exec.ExitError + if errors.As(execErr, &exitErr) && stderrWriter == nil { + execErr = errors.Wrap(ErrFailed, string(exitErr.Stderr)) + } + } + } + errChan <- execErr + }() + return errChan, nil +} diff --git a/pkg/kbagent/service/probe.go b/pkg/kbagent/service/probe.go new file mode 100644 index 00000000000..c8a54c2d300 --- /dev/null +++ b/pkg/kbagent/service/probe.go @@ -0,0 +1,194 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package service + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "strings" + "time" + + "github.com/go-logr/logr" + "golang.org/x/exp/maps" + + "github.com/apecloud/kubeblocks/pkg/kbagent/proto" + "github.com/apecloud/kubeblocks/pkg/kbagent/util" +) + +const ( + probeServiceName = "Probe" + probeServiceVersion = "v1.0" + defaultProbePeriodSeconds = 60 +) + +func newProbeService(logger logr.Logger, actionService *actionService, probes []proto.Probe) (*probeService, error) { + sp := &probeService{ + logger: logger, + actionService: actionService, + probes: make(map[string]*proto.Probe), + runners: make(map[string]*probeRunner), + } + for i, p := range probes { + if _, ok := actionService.actions[p.Action]; !ok { + return nil, fmt.Errorf("probe %s has no action defined", p.Action) + } + sp.probes[p.Action] = &probes[i] + } + logger.Info(fmt.Sprintf("create service %s", sp.Kind()), "probes", strings.Join(maps.Keys(sp.probes), ",")) + return sp, nil +} + +type probeService struct { + logger logr.Logger + actionService *actionService + probes map[string]*proto.Probe + runners map[string]*probeRunner +} + +var _ Service = &probeService{} + +func (s *probeService) Kind() string { + return probeServiceName +} + +func (s *probeService) Version() string { + return probeServiceVersion +} + +func (s *probeService) Start() error { + for name := range s.probes { + runner := &probeRunner{ + logger: s.logger.WithValues("probe", name), + actionService: s.actionService, + } + go runner.run(s.probes[name]) + s.runners[name] = runner + } + return nil +} + +func (s *probeService) Decode(payload []byte) (interface{}, error) { + return nil, ErrNotImplemented +} + +func (s *probeService) HandleRequest(ctx context.Context, req interface{}) ([]byte, error) { + return nil, ErrNotImplemented +} + +type probeRunner struct { + logger logr.Logger + actionService *actionService + ticker *time.Ticker + succeedCount int64 + failedCount int64 + latestOutput []byte +} + +func (r *probeRunner) run(probe *proto.Probe) { + r.logger.Info("probe started", "config", probe) + + if probe.InitialDelaySeconds > 0 { + time.Sleep(time.Duration(probe.InitialDelaySeconds) * time.Second) + } + + if probe.PeriodSeconds <= 0 { + probe.PeriodSeconds = defaultProbePeriodSeconds + } + r.ticker = time.NewTicker(time.Duration(probe.PeriodSeconds) * time.Second) + defer r.ticker.Stop() + + r.runLoop(probe) +} + +func (r *probeRunner) runLoop(probe *proto.Probe) { + for range r.ticker.C { + output, err := r.runOnce(probe) + if err == nil { + r.succeedCount++ + r.failedCount = 0 + } else { + r.succeedCount = 0 + r.failedCount++ + } + + r.report(probe, output, err) + + if succeed, _ := r.succeed(probe); succeed && !reflect.DeepEqual(output, r.latestOutput) { + r.latestOutput = output + } + } +} + +func (r *probeRunner) runOnce(probe *proto.Probe) ([]byte, error) { + return r.actionService.HandleRequest(context.Background(), &proto.ActionRequest{Action: probe.Action}) +} + +func (r *probeRunner) report(probe *proto.Probe, output []byte, err error) { + succeed, thresholdPoint := r.succeed(probe) + if succeed && thresholdPoint || + succeed && !thresholdPoint && !reflect.DeepEqual(output, r.latestOutput) { + r.sendEvent(probe.Action, 0, output, "") + } + if r.fail(probe) { + r.sendEvent(probe.Action, -1, r.latestOutput, err.Error()) + } +} + +func (r *probeRunner) succeed(probe *proto.Probe) (bool, bool) { + if r.succeedCount > 0 { + successThreshold := probe.SuccessThreshold + if successThreshold <= 0 { + successThreshold = 1 + } + return r.succeedCount >= int64(successThreshold), r.succeedCount == int64(successThreshold) + } + return false, false +} + +func (r *probeRunner) fail(probe *proto.Probe) bool { + if r.failedCount > 0 { + failureThreshold := probe.FailureThreshold + if failureThreshold <= 0 { + failureThreshold = 1 + } + return r.failedCount >= int64(failureThreshold) + } + return false +} + +func (r *probeRunner) sendEvent(probe string, code int32, output []byte, message string) { + prefixLen := min(len(output), 32) + r.logger.Info("send probe event", "code", code, "output", string(output[:prefixLen]), "message", message) + + eventMsg := &proto.ProbeEvent{ + Probe: probe, + Code: code, + Message: message, + Output: output, + } + msg, err := json.Marshal(&eventMsg) + if err != nil { + r.logger.Error(err, "failed to marshal probe event") + return + } + util.SendEventWithMessage(&r.logger, probe, string(msg)) +} diff --git a/pkg/kbagent/service/service.go b/pkg/kbagent/service/service.go new file mode 100644 index 00000000000..f2e4d0ea744 --- /dev/null +++ b/pkg/kbagent/service/service.go @@ -0,0 +1,62 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package service + +import ( + "context" + "errors" + + "github.com/go-logr/logr" + + "github.com/apecloud/kubeblocks/pkg/kbagent/proto" +) + +var ( + ErrNotDefined = errors.New("NotDefined") + ErrNotImplemented = errors.New("NotImplemented") + ErrInProgress = errors.New("InProgress") + ErrBusy = errors.New("busy") + ErrTimeout = errors.New("timeout") + ErrFailed = errors.New("failed") + ErrInternalError = errors.New("InternalError") +) + +type Service interface { + Kind() string + Version() string + + Start() error + + Decode([]byte) (interface{}, error) + + HandleRequest(ctx context.Context, req interface{}) ([]byte, error) +} + +func New(logger logr.Logger, actions []proto.Action, probes []proto.Probe) ([]Service, error) { + sa, err := newActionService(logger, actions) + if err != nil { + return nil, err + } + sp, err := newProbeService(logger, sa, probes) + if err != nil { + return nil, err + } + return []Service{sa, sp}, nil +} diff --git a/pkg/kbagent/setup.go b/pkg/kbagent/setup.go new file mode 100644 index 00000000000..c49afc9601b --- /dev/null +++ b/pkg/kbagent/setup.go @@ -0,0 +1,104 @@ +/* +Copyright (C) 2022-2024 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package util + +import ( + "encoding/json" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + + "github.com/apecloud/kubeblocks/pkg/kbagent/proto" + "github.com/apecloud/kubeblocks/pkg/kbagent/service" + "github.com/apecloud/kubeblocks/pkg/kbagent/util" +) + +const ( + actionEnvName = "KB_AGENT_ACTION" + probeEnvName = "KB_AGENT_PROBE" +) + +func BuildStartupEnvs(actions []proto.Action, probes []proto.Probe) ([]corev1.EnvVar, error) { + da, dp, err := serializeActionNProbe(actions, probes) + if err != nil { + return nil, err + } + return []corev1.EnvVar{ + { + Name: actionEnvName, + Value: da, + }, + { + Name: probeEnvName, + Value: dp, + }, + }, nil +} + +func Initialize(logger logr.Logger, envs []string) ([]service.Service, error) { + da, dp := getActionNProbeEnvValue(envs) + if len(da) == 0 { + return nil, nil + } + + actions, probes, err := deserializeActionNProbe(da, dp) + if err != nil { + return nil, err + } + + return service.New(logger, actions, probes) +} + +func getActionNProbeEnvValue(envs []string) (string, string) { + envVars := util.EnvL2M(envs) + da, ok := envVars[actionEnvName] + if !ok { + return "", "" + } + dp, ok := envVars[probeEnvName] + if !ok { + return da, "" + } + return da, dp +} + +func serializeActionNProbe(actions []proto.Action, probes []proto.Probe) (string, string, error) { + da, err := json.Marshal(actions) + if err != nil { + return "", "", nil + } + dp, err := json.Marshal(probes) + if err != nil { + return "", "", nil + } + return string(da), string(dp), nil +} + +func deserializeActionNProbe(da, dp string) ([]proto.Action, []proto.Probe, error) { + actions := make([]proto.Action, 0) + if err := json.Unmarshal([]byte(da), &actions); err != nil { + return nil, nil, err + } + probes := make([]proto.Probe, 0) + if err := json.Unmarshal([]byte(dp), &probes); err != nil { + return nil, nil, err + } + return actions, probes, nil +} diff --git a/pkg/kb_agent/util/k8s_client.go b/pkg/kbagent/util/env.go similarity index 63% rename from pkg/kb_agent/util/k8s_client.go rename to pkg/kbagent/util/env.go index 4e5e89e4d55..de94d709252 100644 --- a/pkg/kb_agent/util/k8s_client.go +++ b/pkg/kbagent/util/env.go @@ -20,21 +20,27 @@ along with this program. If not, see . package util import ( - "github.com/pkg/errors" - "k8s.io/client-go/kubernetes" - ctlruntime "sigs.k8s.io/controller-runtime" + "strings" ) -// GetClientSet returns a kubernetes clientSet. -func GetClientSet() (*kubernetes.Clientset, error) { - restConfig, err := ctlruntime.GetConfig() - if err != nil { - return nil, errors.Wrap(err, "get kubeConfig failed") - } - clientSet, err := kubernetes.NewForConfig(restConfig) - if err != nil { - return nil, err +func EnvM2L(m map[string]string) []string { + l := make([]string, 0) + for k, v := range m { + l = append(l, k+"="+v) } + return l +} - return clientSet, nil +func EnvL2M(l []string) map[string]string { + m := make(map[string]string, 0) + for _, p := range l { + kv := strings.SplitN(p, "=", 2) + if len(kv) == 2 { + m[kv[0]] = kv[1] + } + if len(kv) == 1 { + m[kv[0]] = "" + } + } + return m } diff --git a/pkg/kb_agent/util/event.go b/pkg/kbagent/util/event.go similarity index 60% rename from pkg/kb_agent/util/event.go rename to pkg/kbagent/util/event.go index 2706927dad3..9a8b78d89a9 100644 --- a/pkg/kb_agent/util/event.go +++ b/pkg/kbagent/util/event.go @@ -21,56 +21,43 @@ package util import ( "context" - "encoding/json" - "errors" "fmt" "os" "time" + "github.com/go-logr/logr" + "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/client-go/kubernetes" ctlruntime "sigs.k8s.io/controller-runtime" "github.com/apecloud/kubeblocks/pkg/constant" ) -var logger = ctlruntime.Log.WithName("event") -var EventSendMaxAttempts = 30 -var EventSendPeriod = 10 * time.Second - -func SentEventForProbe(ctx context.Context, msg ActionMessage) error { - logger.Info(fmt.Sprintf("send event: %v", msg)) - action := msg.GetAction() - if action == "" { - return errors.New("action is unset") - } - event, err := CreateEvent(action, msg) - if err != nil { - logger.Info("create event failed", "error", err.Error()) - return err - } +const ( + sendEventMaxAttempts = 30 + sendEventRetryInterval = 10 * time.Second +) +func SendEventWithMessage(logger *logr.Logger, reason string, message string) { go func() { - _ = SendEvent(ctx, event) + event := createEvent(reason, message) + err := sendEvent(event) + if logger != nil && err != nil { + logger.Error(err, "send event failed") + } }() - - return nil } -func CreateEvent(reason string, msg ActionMessage) (*corev1.Event, error) { - // get pod object +func createEvent(reason string, message string) *corev1.Event { podName := os.Getenv(constant.KBEnvPodName) podUID := os.Getenv(constant.KBEnvPodUID) nodeName := os.Getenv(constant.KBEnvNodeName) namespace := os.Getenv(constant.KBEnvNamespace) - data, err := json.Marshal(msg) - if err != nil { - return nil, err - } - - event := &corev1.Event{ + return &corev1.Event{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s.%s", podName, rand.String(16)), Namespace: namespace, @@ -80,41 +67,48 @@ func CreateEvent(reason string, msg ActionMessage) (*corev1.Event, error) { Namespace: namespace, Name: podName, UID: types.UID(podUID), - FieldPath: "spec.containers{kb-agent}", + FieldPath: "spec.containers{kbagent}", }, Reason: reason, - Message: string(data), + Message: message, Source: corev1.EventSource{ - Component: "kb-agent", + Component: "kbagent", Host: nodeName, }, FirstTimestamp: metav1.Now(), LastTimestamp: metav1.Now(), EventTime: metav1.NowMicro(), - ReportingController: "kb-agent", + ReportingController: "kbagent", ReportingInstance: podName, Action: reason, Type: "Normal", } - return event, nil } -func SendEvent(ctx context.Context, event *corev1.Event) error { - ctx1 := context.Background() - clientset, err := GetClientSet() +func sendEvent(event *corev1.Event) error { + clientSet, err := getK8sClientSet() if err != nil { - logger.Info("k8s client create failed", "error", err.Error()) return err } namespace := os.Getenv(constant.KBEnvNamespace) - for i := 0; i < EventSendMaxAttempts; i++ { - _, err = clientset.CoreV1().Events(namespace).Create(ctx1, event, metav1.CreateOptions{}) + for i := 0; i < sendEventMaxAttempts; i++ { + _, err = clientSet.CoreV1().Events(namespace).Create(context.Background(), event, metav1.CreateOptions{}) if err == nil { - logger.Info("send event success", "message", event.Message) - break + return nil } - logger.Info("send event failed", "error", err.Error()) - time.Sleep(EventSendPeriod) + time.Sleep(sendEventRetryInterval) } return err } + +func getK8sClientSet() (*kubernetes.Clientset, error) { + restConfig, err := ctlruntime.GetConfig() + if err != nil { + return nil, errors.Wrap(err, "get kubeConfig failed") + } + clientSet, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return nil, err + } + return clientSet, nil +} diff --git a/pkg/testutil/apps/constant.go b/pkg/testutil/apps/constant.go index 4ae38fa0d51..251b7ce8fc1 100644 --- a/pkg/testutil/apps/constant.go +++ b/pkg/testutil/apps/constant.go @@ -20,6 +20,8 @@ along with this program. If not, see . package apps import ( + "fmt" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/util/intstr" @@ -58,6 +60,15 @@ var ( defaultLifecycleActionHandler = &appsv1alpha1.LifecycleActionHandler{ BuiltinHandler: &defaultBuiltinHandler, } + newLifecycleActionHandler = func(name string) *appsv1alpha1.LifecycleActionHandler { + return &appsv1alpha1.LifecycleActionHandler{ + CustomHandler: &appsv1alpha1.Action{ + Exec: &appsv1alpha1.ExecAction{ + Command: []string{"/bin/sh", "-c", fmt.Sprintf("echo %s", name)}, + }, + }, + } + } zeroResRequirements = corev1.ResourceRequirements{ Limits: map[corev1.ResourceName]resource.Quantity{ @@ -239,13 +250,13 @@ var ( }, Switchover: nil, MemberJoin: defaultLifecycleActionHandler, - MemberLeave: defaultLifecycleActionHandler, + MemberLeave: newLifecycleActionHandler("member-leave"), Readonly: defaultLifecycleActionHandler, Readwrite: defaultLifecycleActionHandler, DataDump: defaultLifecycleActionHandler, DataLoad: defaultLifecycleActionHandler, Reconfigure: defaultLifecycleActionHandler, - AccountProvision: defaultLifecycleActionHandler, + AccountProvision: newLifecycleActionHandler("account-provision"), }, }