diff --git a/test/config/config.go b/test/config/config.go index 0ddd20c517..ccbb41d7d1 100644 --- a/test/config/config.go +++ b/test/config/config.go @@ -17,6 +17,7 @@ import ( "os" "path/filepath" "strconv" + "strings" "time" "github.com/alibaba/ilogtail/pkg/logger" @@ -46,7 +47,6 @@ type Config struct { AccessKeySecret string `mapstructure:"access_key_secret" yaml:"access_key_secret"` Endpoint string `mapstructure:"endpoint" yaml:"endpoint"` Aliuid string `mapstructure:"aliuid" yaml:"aliuid"` - QueryEndpoint string `mapstructure:"query_endpoint" yaml:"query_endpoint"` Region string `mapstructure:"region" yaml:"region"` RetryTimeout time.Duration `mapstructure:"retry_timeout" yaml:"retry_timeout"` } @@ -94,7 +94,6 @@ func ParseConfig() { TestConfig.AccessKeySecret = os.Getenv("ACCESS_KEY_SECRET") TestConfig.Endpoint = os.Getenv("ENDPOINT") TestConfig.Aliuid = os.Getenv("ALIUID") - TestConfig.QueryEndpoint = os.Getenv("QUERY_ENDPOINT") TestConfig.Region = os.Getenv("REGION") timeout, err := strconv.ParseInt(os.Getenv("RETRY_TIMEOUT"), 10, 64) if err != nil { @@ -102,3 +101,11 @@ func ParseConfig() { } TestConfig.RetryTimeout = time.Duration(timeout) * time.Second } + +func GetQueryEndpoint() string { + idx := strings.Index(TestConfig.Endpoint, "-intranet") + if idx == -1 { + return TestConfig.Endpoint + } + return TestConfig.Endpoint[:idx] + TestConfig.Endpoint[idx+9:] +} diff --git a/test/config/context.go b/test/config/context.go index d6f4c0d57a..6181498732 100644 --- a/test/config/context.go +++ b/test/config/context.go @@ -23,4 +23,5 @@ const ( CurrentWorkingDeploymentKey ContextKey = "currentWorkingDeployment" QueryKey ContextKey = "query" AgentPIDKey ContextKey = "agentPID" + EndpointIPKey ContextKey = "endpointIP" ) diff --git a/test/engine/cleanup/chaos.go b/test/engine/cleanup/chaos.go new file mode 100644 index 0000000000..710f9ac874 --- /dev/null +++ b/test/engine/cleanup/chaos.go @@ -0,0 +1,71 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package cleanup + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + + "github.com/alibaba/ilogtail/test/engine/setup" +) + +type ChaosStatus struct { + Code int `json:"code"` + Success bool `json:"success"` + Result []map[string]string `json:"result"` +} + +func DestoryAllChaos(ctx context.Context) (context.Context, error) { + switch setup.Env.GetType() { + case "host": + command := "/opt/chaosblade/blade status --type create --status Success" + response, err := setup.Env.ExecOnLogtail(command) + if err != nil { + return ctx, err + } + var status ChaosStatus + if err = json.Unmarshal([]byte(response), &status); err != nil { + return ctx, err + } + for _, result := range status.Result { + command = "/opt/chaosblade/blade destroy " + result["Uid"] + setup.Env.ExecOnLogtail(command) + } + case "daemonset", "deployment": + k8sEnv := setup.Env.(*setup.K8sEnv) + chaosDir := filepath.Join("test_cases", "chaos") + err := filepath.Walk(chaosDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + if filepath.Ext(path) != ".yaml" { + return nil + } + return k8sEnv.Delete(path[len("test_cases/"):]) + }) + if err != nil { + return ctx, err + } + // delete chaosDir + if err = os.RemoveAll(chaosDir); err != nil { + return ctx, err + } + } + return ctx, nil +} diff --git a/test/engine/cleanup/helper.go b/test/engine/cleanup/helper.go index c223800104..f6cb2c7c34 100644 --- a/test/engine/cleanup/helper.go +++ b/test/engine/cleanup/helper.go @@ -15,6 +15,7 @@ package cleanup import ( "context" + "fmt" "os" "os/signal" "syscall" @@ -44,10 +45,23 @@ func All() { return } ctx := context.TODO() - _, _ = control.RemoveAllLocalConfig(ctx) - _, _ = AllGeneratedLog(ctx) - _, _ = GoTestCache(ctx) - _, _ = DeleteContainers(ctx) + red := "\033[31m" + reset := "\033[0m" + if _, err := control.RemoveAllLocalConfig(ctx); err != nil { + fmt.Println(red + err.Error() + reset) + } + if _, err := AllGeneratedLog(ctx); err != nil { + fmt.Println(red + err.Error() + reset) + } + if _, err := GoTestCache(ctx); err != nil { + fmt.Println(red + err.Error() + reset) + } + if _, err := DestoryAllChaos(ctx); err != nil { + fmt.Println(red + err.Error() + reset) + } + if _, err := DeleteContainers(ctx); err != nil { + fmt.Println(red + err.Error() + reset) + } if subscriber.TestSubscriber != nil { _ = subscriber.TestSubscriber.Stop() } diff --git a/test/engine/control/agent.go b/test/engine/control/agent.go new file mode 100644 index 0000000000..062e1d0d0e --- /dev/null +++ b/test/engine/control/agent.go @@ -0,0 +1,41 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package control + +import ( + "context" + "fmt" + "strings" + + "github.com/alibaba/ilogtail/test/config" + "github.com/alibaba/ilogtail/test/engine/setup" +) + +func RestartAgent(ctx context.Context) (context.Context, error) { + setup.Env.ExecOnLogtail("/etc/init.d/loongcollectord restart") + return setup.SetAgentPID(ctx) +} + +func ForceRestartAgent(ctx context.Context) (context.Context, error) { + currentPID := ctx.Value(config.AgentPIDKey) + if currentPID != nil { + currentPIDs := strings.Split(currentPID.(string), "\n") + for _, pid := range currentPIDs { + setup.Env.ExecOnLogtail("kill -9 " + pid) + } + } + fmt.Println("No agent pid found, skip force restart") + setup.Env.ExecOnLogtail("/etc/init.d/loongcollectord start") + return setup.SetAgentPID(ctx) +} diff --git a/test/engine/setup/chaos/network.go b/test/engine/setup/chaos/network.go new file mode 100644 index 0000000000..ac713eddfd --- /dev/null +++ b/test/engine/setup/chaos/network.go @@ -0,0 +1,156 @@ +// Copyright 2024 iLogtail Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package chaos + +import ( + "context" + "os" + "path/filepath" + "strconv" + "text/template" + + "github.com/alibaba/ilogtail/test/engine/setup" +) + +const ( + // networkDelayCRD + networkDelayCRDTmpl = ` +apiVersion: chaosblade.io/v1alpha1 +kind: ChaosBlade +metadata: + name: delay-pod-network +spec: + experiments: + - scope: pod + target: network + action: delay + desc: "delay pod network" + matchers: + - name: labels + value: ["{{.PodLabel}}"] + - name: namespace + value: ["kube-system"] + - name: interface + value: ["eth0"] + - name: destination-ip + value: ["{{.Percent}}"] + - name: time + value: ["{{.Time}}"] +` + + // networkLossCRD + networkLossCRDTmpl = ` +apiVersion: chaosblade.io/v1alpha1 +kind: ChaosBlade +metadata: + name: loss-pod-network +spec: + experiments: + - scope: pod + target: network + action: loss + desc: "loss pod network" + matchers: + - name: labels + value: ["{{.PodLabel}}"] + - name: namespace + value: ["kube-system"] + - name: interface + value: ["eth0"] + - name: percent + value: ["{{.Percent}}"] + - name: exclude-port + value: ["22"] + - name: destination-ip + value: ["{{.Ip}}"] +` +) + +func NetworkDelay(ctx context.Context, time int, ip string) (context.Context, error) { + switch setup.Env.GetType() { + case "host": + command := "/opt/chaosblade/blade create network delay --time " + strconv.FormatInt(int64(time), 10) + " --exclude-port 22 --interface eth0 --destination-ip " + ip + _, err := setup.Env.ExecOnLogtail(command) + if err != nil { + return ctx, err + } + case "daemonset", "deployment": + dir := filepath.Join("test_cases", "chaos") + filename := "loss-pod-network.yaml" + _ = os.Mkdir(dir, 0755) + file, err := os.OpenFile(filepath.Join(dir, filename), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) //nolint:gosec + if err != nil { + return ctx, err + } + defer file.Close() //nolint:gosec + + networkDelayCRD, _ := template.New("networkDelay").Parse(networkDelayCRDTmpl) + if err = networkDelayCRD.Execute(file, map[string]string{ + "PodLabel": getLoongCollectorPodLabel(), + "Time": strconv.FormatInt(int64(time), 10), + "Ip": ip, + }); err != nil { + return ctx, err + } + k8sEnv := setup.Env.(*setup.K8sEnv) + if err := k8sEnv.Apply(filepath.Join("chaos", filename)); err != nil { + return ctx, err + } + } + return ctx, nil +} + +func NetworkLoss(ctx context.Context, percentage int, ip string) (context.Context, error) { + switch setup.Env.GetType() { + case "host": + command := "/opt/chaosblade/blade create network loss --percent " + strconv.FormatInt(int64(percentage), 10) + " --exclude-port 22 --interface eth0 --destination-ip " + ip + _, err := setup.Env.ExecOnLogtail(command) + if err != nil { + return ctx, err + } + case "daemonset", "deployment": + dir := filepath.Join("test_cases", "chaos") + filename := "loss-pod-network.yaml" + _ = os.Mkdir(dir, 0755) + file, err := os.OpenFile(filepath.Join(dir, filename), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) //nolint:gosec + if err != nil { + return ctx, err + } + defer file.Close() + + networkLossCRD, _ := template.New("networkLoss").Parse(networkLossCRDTmpl) + if err = networkLossCRD.Execute(file, map[string]string{ + "PodLabel": getLoongCollectorPodLabel(), + "Percent": strconv.FormatInt(int64(percentage), 10), + "Ip": ip, + }); err != nil { + return ctx, err + } + k8sEnv := setup.Env.(*setup.K8sEnv) + if err := k8sEnv.Apply(filepath.Join("chaos", filename)); err != nil { + return ctx, err + } + } + return ctx, nil +} + +func getLoongCollectorPodLabel() string { + var PodLabel string + if setup.Env.GetType() == "daemonset" { + PodLabel = "k8s-app=logtail-ds" + } else if setup.Env.GetType() == "deployment" { + PodLabel = "k8s-app=loongcollector-cluster" + } + return PodLabel +} diff --git a/test/engine/setup/controller/kubernetes.go b/test/engine/setup/controller/kubernetes.go index 39efb316a3..0bd0a0f62e 100644 --- a/test/engine/setup/controller/kubernetes.go +++ b/test/engine/setup/controller/kubernetes.go @@ -230,18 +230,18 @@ func (c *DynamicController) Apply(filePath string) error { } // Apply the object to the Kubernetes cluster - namespace := obj.GetNamespace() - if namespace == "" { - namespace = "default" // Use default namespace if not specified - } - resourceInterface := c.dynamicClient.Resource(mapping.Resource).Namespace(namespace) - if _, err := resourceInterface.Get(context.TODO(), obj.GetName(), metav1.GetOptions{}); err != nil { + resourceInterface := c.dynamicClient.Resource(mapping.Resource) + if oldObj, err := resourceInterface.Get(context.TODO(), obj.GetName(), metav1.GetOptions{}); err != nil { + if !meta.IsNoMatchError(err) { + return err + } // Object does not exist, create it if _, err := resourceInterface.Create(context.TODO(), obj, metav1.CreateOptions{}); err != nil { return err } } else { // Object exists, update it + obj.SetResourceVersion(oldObj.GetResourceVersion()) if _, err := resourceInterface.Update(context.TODO(), obj, metav1.UpdateOptions{}); err != nil { return err } @@ -257,11 +257,7 @@ func (c *DynamicController) Delete(filePath string) error { } // Delete the object from the Kubernetes cluster - namespace := obj.GetNamespace() - if namespace == "" { - namespace = "default" // Use default namespace if not specified - } - resourceInterface := c.dynamicClient.Resource(mapping.Resource).Namespace(namespace) + resourceInterface := c.dynamicClient.Resource(mapping.Resource) if err := resourceInterface.Delete(context.TODO(), obj.GetName(), metav1.DeleteOptions{}); err != nil { return err } diff --git a/test/engine/setup/subscriber/sls.go b/test/engine/setup/subscriber/sls.go index 9d05c56cc6..b817bee8fe 100644 --- a/test/engine/setup/subscriber/sls.go +++ b/test/engine/setup/subscriber/sls.go @@ -29,6 +29,11 @@ flushers: type SLSSubscriber struct { client *sls.Client TelemetryType string + Aliuid string + Region string + Endpoint string + Project string + Logstore string } func (s *SLSSubscriber) Name() string { @@ -65,11 +70,11 @@ func (s *SLSSubscriber) FlusherConfig() string { tpl := template.Must(template.New("slsFlusherConfig").Parse(SLSFlusherConfigTemplate)) var builder strings.Builder _ = tpl.Execute(&builder, map[string]interface{}{ - "Aliuid": config.TestConfig.Aliuid, - "Region": config.TestConfig.Region, - "Endpoint": config.TestConfig.Endpoint, - "Project": config.TestConfig.Project, - "Logstore": config.TestConfig.GetLogstore(s.TelemetryType), + "Aliuid": s.Aliuid, + "Region": s.Region, + "Endpoint": s.Endpoint, + "Project": s.Project, + "Logstore": s.Logstore, "TelemetryType": s.TelemetryType, }) config := builder.String() @@ -135,9 +140,34 @@ func init() { } fmt.Println("create sls subscriber with telemetry type", telemetryType) l := &SLSSubscriber{ - client: createSLSClient(config.TestConfig.AccessKeyID, config.TestConfig.AccessKeySecret, config.TestConfig.QueryEndpoint), + client: createSLSClient(config.TestConfig.AccessKeyID, config.TestConfig.AccessKeySecret, config.GetQueryEndpoint()), TelemetryType: telemetryType, } + if v, ok := spec["aliuid"]; ok { + l.Aliuid = v.(string) + } else { + l.Aliuid = config.TestConfig.Aliuid + } + if v, ok := spec["region"]; ok { + l.Region = v.(string) + } else { + l.Region = config.TestConfig.Region + } + if v, ok := spec["endpoint"]; ok { + l.Endpoint = v.(string) + } else { + l.Endpoint = config.TestConfig.Endpoint + } + if v, ok := spec["project"]; ok { + l.Project = v.(string) + } else { + l.Project = config.TestConfig.Project + } + if v, ok := spec["logstore"]; ok { + l.Logstore = v.(string) + } else { + l.Logstore = config.TestConfig.GetLogstore(telemetryType) + } return l, nil }) doc.Register("subscriber", slsName, new(SLSSubscriber)) diff --git a/test/engine/steps.go b/test/engine/steps.go index 336a5ff8ac..dbbb2524dc 100644 --- a/test/engine/steps.go +++ b/test/engine/steps.go @@ -10,6 +10,7 @@ import ( "github.com/alibaba/ilogtail/test/engine/cleanup" "github.com/alibaba/ilogtail/test/engine/control" "github.com/alibaba/ilogtail/test/engine/setup" + "github.com/alibaba/ilogtail/test/engine/setup/chaos" "github.com/alibaba/ilogtail/test/engine/setup/monitor" "github.com/alibaba/ilogtail/test/engine/setup/subscriber" "github.com/alibaba/ilogtail/test/engine/trigger" @@ -31,6 +32,11 @@ func ScenarioInitializer(ctx *godog.ScenarioContext) { ctx.Given(`^subcribe data from \{(\S+)\} with config`, subscriber.InitSubscriber) ctx.Given(`^mkdir \{(.*)\}`, setup.Mkdir) ctx.Given(`^docker-compose boot type \{(\S+)\}$`, setup.SetDockerComposeBootType) + + // chaos + ctx.Given(`^network delay package \{(\d+)\}ms for ip \{(.*)\}`, chaos.NetworkDelay) + ctx.Given(`^network lost package \{(\d+)\}% for ip \{(.*)\}`, chaos.NetworkLoss) + ctx.Given(`^clean all chaos$`, cleanup.DestoryAllChaos) // ------------------------------------------ // When @@ -42,6 +48,8 @@ func ScenarioInitializer(ctx *godog.ScenarioContext) { ctx.When(`^query through \{(.*)\}`, control.SetQuery) ctx.When(`^apply yaml \{(.*)\} to k8s`, control.ApplyYaml) ctx.When(`^delete yaml \{(.*)\} from k8s`, control.DeleteYaml) + ctx.When(`^restart agent`, control.RestartAgent) + ctx.When(`^force restart agent`, control.ForceRestartAgent) // generate ctx.When(`^begin trigger`, trigger.BeginTrigger) diff --git a/test/requirements.txt b/test/requirements.txt index ea45cd03b7..02ac307220 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1 +1 @@ -Faker \ No newline at end of file +Faker==30.8.2 \ No newline at end of file