Merge branch 'master' into fix-empty-label

components/playground/instance/pd.go

@@ -89,6 +89,15 @@ func (inst *PDInstance) Name() string {
 
 // Start calls set inst.cmd and Start
 func (inst *PDInstance) Start(ctx context.Context, version utils.Version) error {
+	configPath := filepath.Join(inst.Dir, "pd.toml")
+	if err := prepareConfig(
+		configPath,
+		inst.ConfigPath,
+		inst.getConfig(),
+	); err != nil {
+		return err
+	}
+
 	uid := inst.Name()
 	var args []string
 	switch inst.Role {
@@ -98,16 +107,15 @@ func (inst *PDInstance) Start(ctx context.Context, version utils.Version) error
 		}
 		args = append(args, []string{
 			"--name=" + uid,
+			fmt.Sprintf("--config=%s", configPath),
 			fmt.Sprintf("--data-dir=%s", filepath.Join(inst.Dir, "data")),
 			fmt.Sprintf("--peer-urls=http://%s", utils.JoinHostPort(inst.Host, inst.Port)),
 			fmt.Sprintf("--advertise-peer-urls=http://%s", utils.JoinHostPort(AdvertiseHost(inst.Host), inst.Port)),
 			fmt.Sprintf("--client-urls=http://%s", utils.JoinHostPort(inst.Host, inst.StatusPort)),
 			fmt.Sprintf("--advertise-client-urls=http://%s", utils.JoinHostPort(AdvertiseHost(inst.Host), inst.StatusPort)),
 			fmt.Sprintf("--log-file=%s", inst.LogFile()),
 		}...)
-		if inst.ConfigPath != "" {
-			args = append(args, fmt.Sprintf("--config=%s", inst.ConfigPath))
-		}
+
 		switch {
 		case len(inst.initEndpoints) > 0:
 			endpoints := make([]string, 0)

components/playground/instance/pd_config.go

@@ -0,0 +1,20 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package instance
+
+func (inst *PDInstance) getConfig() map[string]any {
+	config := make(map[string]any)
+	config["schedule.patrol-region-interval"] = "100ms"
+	return config
+}

components/playground/instance/tiflash.go

@@ -20,6 +20,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/pingcap/errors"
 	tiupexec "github.com/pingcap/tiup/pkg/exec"
 	"github.com/pingcap/tiup/pkg/tidbver"
 	"github.com/pingcap/tiup/pkg/utils"
@@ -131,26 +132,35 @@ func (inst *TiFlashInstance) Start(ctx context.Context, version utils.Version) e
 
 	args := []string{
 		"server",
-	}
-	args = append(args,
 		fmt.Sprintf("--config-file=%s", configPath),
 		"--",
-		fmt.Sprintf("--tmp_path=%s", filepath.Join(inst.Dir, "tmp")),
-		fmt.Sprintf("--path=%s", filepath.Join(inst.Dir, "data")),
-		fmt.Sprintf("--listen_host=%s", inst.Host),
-		fmt.Sprintf("--logger.log=%s", inst.LogFile()),
-		fmt.Sprintf("--logger.errorlog=%s", filepath.Join(inst.Dir, "tiflash_error.log")),
-		fmt.Sprintf("--status.metrics_port=%d", inst.StatusPort),
-		fmt.Sprintf("--flash.service_addr=%s", utils.JoinHostPort(AdvertiseHost(inst.Host), inst.ServicePort)),
-		fmt.Sprintf("--raft.pd_addr=%s", strings.Join(endpoints, ",")),
-		fmt.Sprintf("--flash.proxy.addr=%s", utils.JoinHostPort(inst.Host, inst.ProxyPort)),
-		fmt.Sprintf("--flash.proxy.advertise-addr=%s", utils.JoinHostPort(AdvertiseHost(inst.Host), inst.ProxyPort)),
-		fmt.Sprintf("--flash.proxy.status-addr=%s", utils.JoinHostPort(inst.Host, inst.ProxyStatusPort)),
-		fmt.Sprintf("--flash.proxy.data-dir=%s", filepath.Join(inst.Dir, "proxy_data")),
-		fmt.Sprintf("--flash.proxy.log-file=%s", filepath.Join(inst.Dir, "tiflash_tikv.log")),
-	)
-
-	var err error
+	}
+	runtimeConfig := [][]string{
+		{"path", filepath.Join(inst.Dir, "data")},
+		{"listen_host", inst.Host},
+		{"logger.log", inst.LogFile()},
+		{"logger.errorlog", filepath.Join(inst.Dir, "tiflash_error.log")},
+		{"status.metrics_port", fmt.Sprintf("%d", inst.StatusPort)},
+		{"flash.service_addr", utils.JoinHostPort(AdvertiseHost(inst.Host), inst.ServicePort)},
+		{"raft.pd_addr", strings.Join(endpoints, ",")},
+		{"flash.proxy.addr", utils.JoinHostPort(inst.Host, inst.ProxyPort)},
+		{"flash.proxy.advertise-addr", utils.JoinHostPort(AdvertiseHost(inst.Host), inst.ProxyPort)},
+		{"flash.proxy.status-addr", utils.JoinHostPort(inst.Host, inst.ProxyStatusPort)},
+		{"flash.proxy.data-dir", filepath.Join(inst.Dir, "proxy_data")},
+		{"flash.proxy.log-file", filepath.Join(inst.Dir, "tiflash_tikv.log")},
+	}
+	userConfig, err := unmarshalConfig(configPath)
+	if err != nil {
+		return errors.Trace(err)
+	}
+	fmt.Println("userConfig", userConfig)
+	for _, arg := range runtimeConfig {
+		// if user has set the config, skip it
+		if !isKeyPresentInMap(userConfig, arg[0]) {
+			args = append(args, fmt.Sprintf("--%s=%s", arg[0], arg[1]))
+		}
+	}
+
 	if inst.BinPath, err = tiupexec.PrepareBinary("tiflash", version, inst.BinPath); err != nil {
 		return err
 	}
@@ -160,6 +170,24 @@ func (inst *TiFlashInstance) Start(ctx context.Context, version utils.Version) e
 	return inst.Process.Start()
 }
 
+func isKeyPresentInMap(m map[string]any, key string) bool {
+	keys := strings.Split(key, ".")
+	currentMap := m
+
+	for i := 0; i < len(keys); i++ {
+		if _, ok := currentMap[keys[i]]; !ok {
+			return false
+		}
+
+		// If the current value is a nested map, update the current map to the nested map
+		if innerMap, ok := currentMap[keys[i]].(map[string]any); ok {
+			currentMap = innerMap
+		}
+	}
+
+	return true
+}
+
 // Component return the component name.
 func (inst *TiFlashInstance) Component() string {
 	return "tiflash"

components/playground/instance/tiflash_config.go

@@ -28,6 +28,7 @@ func (inst *TiFlashInstance) getConfig() map[string]any {
 	config := make(map[string]any)
 
 	config["flash.proxy.config"] = filepath.Join(inst.Dir, "tiflash_proxy.toml")
+	config["logger.level"] = "debug"
 
 	if inst.Role == TiFlashRoleDisaggWrite {
 		config["storage.s3.endpoint"] = inst.DisaggOpts.S3Endpoint

components/playground/main.go

@@ -414,17 +414,17 @@ func populateDefaultOpt(flagSet *pflag.FlagSet) error {
 		defaultInt(&options.PD.Num, "pd", 1)
 	case "ms":
 		defaultInt(&options.PDAPI.Num, "pd.api", 1)
-		defaultStr(&options.PDAPI.BinPath, "pd.api.binpath", options.PDAPI.BinPath)
-		defaultStr(&options.PDAPI.ConfigPath, "pd.api.config", options.PDAPI.ConfigPath)
+		defaultStr(&options.PDAPI.BinPath, "pd.api.binpath", options.PD.BinPath)
+		defaultStr(&options.PDAPI.ConfigPath, "pd.api.config", options.PD.ConfigPath)
 		defaultInt(&options.PDTSO.Num, "pd.tso", 1)
-		defaultStr(&options.PDTSO.BinPath, "pd.tso.binpath", options.PDTSO.BinPath)
-		defaultStr(&options.PDTSO.ConfigPath, "pd.tso.config", options.PDTSO.ConfigPath)
+		defaultStr(&options.PDTSO.BinPath, "pd.tso.binpath", options.PD.BinPath)
+		defaultStr(&options.PDTSO.ConfigPath, "pd.tso.config", options.PD.ConfigPath)
 		defaultInt(&options.PDScheduling.Num, "pd.scheduling", 1)
-		defaultStr(&options.PDScheduling.BinPath, "pd.scheduling.binpath", options.PDScheduling.BinPath)
-		defaultStr(&options.PDScheduling.ConfigPath, "pd.scheduling.config", options.PDScheduling.ConfigPath)
+		defaultStr(&options.PDScheduling.BinPath, "pd.scheduling.binpath", options.PD.BinPath)
+		defaultStr(&options.PDScheduling.ConfigPath, "pd.scheduling.config", options.PD.ConfigPath)
 		defaultInt(&options.PDRM.Num, "pd.rm", 1)
-		defaultStr(&options.PDRM.BinPath, "pd.rm.binpath", options.PDRM.BinPath)
-		defaultStr(&options.PDRM.ConfigPath, "pd.rm.config", options.PDRM.ConfigPath)
+		defaultStr(&options.PDRM.BinPath, "pd.rm.binpath", options.PD.BinPath)
+		defaultStr(&options.PDRM.ConfigPath, "pd.rm.config", options.PD.ConfigPath)
 	default:
 		return errors.Errorf("Unknown --pd.mode %s", options.PDMode)
 	}

pkg/cluster/spec/tiflash.go

@@ -537,6 +537,7 @@ func (i *TiFlashInstance) initTiFlashConfig(ctx context.Context, version string,
 		daemonConfig = `application.runAsDaemon: true`
 		markCacheSize = `mark_cache_size: 5368709120`
 	}
+
 	err = yaml.Unmarshal([]byte(fmt.Sprintf(`
 server_configs:
   tiflash:
@@ -559,11 +560,9 @@ server_configs:
     logger.errorlog: "%[2]s/tiflash_error.log"
     logger.log: "%[2]s/tiflash.log"
     logger.count: 20
-    logger.level: "debug"
     logger.size: "1000M"
     %[13]s
     raft.pd_addr: "%[9]s"
-    profiles.default.max_memory_usage: 0
     %[12]s
     %[14]s
 `,

pkg/cluster/spec/tiproxy.go

@@ -136,8 +136,15 @@ func (c *TiProxyComponent) Source() string {
 
 // CalculateVersion implements the Component interface
 func (c *TiProxyComponent) CalculateVersion(clusterVersion string) string {
-	// always not follow global version, use ""(latest) by default
 	version := c.Topology.ComponentVersions.TiProxy
+	if version == "" {
+		// always not follow global version
+		// because tiproxy version is different from clusterVersion
+		// but "nightly" is effective
+		if clusterVersion == "nightly" {
+			version = clusterVersion
+		}
+	}
 	return version
 }
 
@@ -213,18 +220,16 @@ func (i *TiProxyInstance) checkConfig(
 ) map[string]any {
 	topo := i.topo.(*Specification)
 	spec := i.InstanceSpec.(*TiProxySpec)
-	enableTLS := topo.GlobalOptions.TLSEnabled
 
 	if cfg == nil {
 		cfg = make(map[string]any)
 	}
 
 	pds := []string{}
 	for _, pdspec := range topo.PDServers {
-		pds = append(pds, pdspec.GetAdvertiseClientURL(enableTLS))
+		pds = append(pds, utils.JoinHostPort(pdspec.Host, pdspec.ClientPort))
 	}
 	cfg["proxy.pd-addrs"] = strings.Join(pds, ",")
-	cfg["proxy.require-backend-tls"] = false
 	cfg["proxy.addr"] = utils.JoinHostPort(i.GetListenHost(), i.GetPort())
 	cfg["api.addr"] = utils.JoinHostPort(i.GetListenHost(), spec.StatusPort)
 	cfg["log.log-file.filename"] = filepath.Join(paths.Log, "tiproxy.log")
@@ -269,7 +274,7 @@ func (i *TiProxyInstance) InitConfig(
 	}
 
 	var err error
-	instanceConfig, err = i.setTLSConfig(ctx, false, instanceConfig, paths)
+	instanceConfig, err = i.setTLSConfig(ctx, topo.GlobalOptions.TLSEnabled, instanceConfig, paths)
 	if err != nil {
 		return err
 	}
@@ -287,12 +292,14 @@ func (i *TiProxyInstance) setTLSConfig(ctx context.Context, enableTLS bool, conf
 		configs["security.cluster-tls.cert"] = fmt.Sprintf("%s/tls/%s.crt", paths.Deploy, i.Role())
 		configs["security.cluster-tls.key"] = fmt.Sprintf("%s/tls/%s.pem", paths.Deploy, i.Role())
 
-		configs["security.server-tls.ca"] = fmt.Sprintf("%s/tls/%s", paths.Deploy, TLSCACert)
-		configs["security.server-tls.cert"] = fmt.Sprintf("%s/tls/%s.crt", paths.Deploy, i.Role())
-		configs["security.server-tls.key"] = fmt.Sprintf("%s/tls/%s.pem", paths.Deploy, i.Role())
-		configs["security.server-tls.skip-ca"] = true
+		configs["security.server-http-tls.ca"] = fmt.Sprintf("%s/tls/%s", paths.Deploy, TLSCACert)
+		configs["security.server-http-tls.cert"] = fmt.Sprintf("%s/tls/%s.crt", paths.Deploy, i.Role())
+		configs["security.server-http-tls.key"] = fmt.Sprintf("%s/tls/%s.pem", paths.Deploy, i.Role())
+		configs["security.server-http-tls.skip-ca"] = true
 
 		configs["security.sql-tls.ca"] = fmt.Sprintf("%s/tls/%s", paths.Deploy, TLSCACert)
+		configs["security.sql-tls.cert"] = fmt.Sprintf("%s/tls/%s.crt", paths.Deploy, i.Role())
+		configs["security.sql-tls.key"] = fmt.Sprintf("%s/tls/%s.pem", paths.Deploy, i.Role())
 	} else {
 		// drainer tls config list
 		tlsConfigs := []string{
@@ -303,15 +310,21 @@ func (i *TiProxyInstance) setTLSConfig(ctx context.Context, enableTLS bool, conf
 			"security.server-tls.cert",
 			"security.server-tls.key",
 			"security.server-tls.skip-ca",
+			"security.server-http-tls.ca",
+			"security.server-http-tls.cert",
+			"security.server-http-tls.key",
+			"security.server-http-tls.skip-ca",
 			"security.sql-tls.ca",
+			"security.sql-tls.cert",
+			"security.sql-tls.key",
 		}
 		// delete TLS configs
 		for _, config := range tlsConfigs {
 			delete(configs, config)
 		}
 	}
 
-	return nil, nil
+	return configs, nil
 }
 
 var _ RollingUpdateInstance = &TiProxyInstance{}

pkg/cluster/spec/validate.go

@@ -938,6 +938,7 @@ func (s *Specification) validateTLSEnabled() error {
 			ComponentTiDB,
 			ComponentTiKV,
 			ComponentTiFlash,
+			ComponentTiProxy,
 			ComponentPump,
 			ComponentDrainer,
 			ComponentCDC,

pkg/repository/clone_mirror.go

@@ -423,10 +423,6 @@ func checkVersion(options CloneOptions, versions set.StringSet, version string)
 func combineVersions(componentVersions *[]string,
 	manifest *v1manifest.Component, oss, archs,
 	globalVersions []string) (set.StringSet, error) {
-	if (componentVersions == nil || len(*componentVersions) < 1) && len(globalVersions) < 1 {
-		return nil, errors.New("no version specified")
-	}
-
 	result := set.NewStringSet()
 	for _, os := range oss {
 		for _, arch := range archs {