Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[DNM]: Ssl content debug #11908

Open
wants to merge 47 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
47 commits
Select commit Hold shift + click to select a range
39eafe7
generate api client and server
River2000i Dec 5, 2024
5c1f5a0
use different tls config
River2000i Dec 5, 2024
ef1e4fb
add ca2 tls key
River2000i Dec 6, 2024
cf6e633
Merge branch 'master' of github.com:pingcap/tiflow into dmMultiSecuri…
River2000i Dec 9, 2024
c2974c3
add comment and fix
River2000i Dec 9, 2024
cd45903
add test
River2000i Dec 9, 2024
17331f5
add ut
River2000i Dec 9, 2024
ee0bd4c
add ut
River2000i Dec 10, 2024
c853e28
add test
River2000i Dec 10, 2024
52d71ff
fix test
River2000i Dec 10, 2024
729d265
fix test
River2000i Dec 11, 2024
6a00b34
fix test
River2000i Dec 11, 2024
d1c5ea1
add test
River2000i Dec 12, 2024
dcdac1e
fix certificates
River2000i Dec 12, 2024
bf1992d
add test
River2000i Dec 12, 2024
47b2015
add test
River2000i Dec 12, 2024
64de0cf
fix test
River2000i Dec 12, 2024
5df2307
fix test
River2000i Dec 12, 2024
ec1b94f
fmt
River2000i Dec 12, 2024
c8aea05
fix test
River2000i Dec 13, 2024
0650f6b
fix test
River2000i Dec 13, 2024
3526f10
support set ssl by file path
River2000i Dec 16, 2024
60df0b1
Merge remote-tracking branch 'upstream/master' into dmMultiSecurityCo…
River2000i Dec 16, 2024
645b726
fmt
River2000i Dec 16, 2024
f87a110
fmt
River2000i Dec 16, 2024
edf188f
fix test
River2000i Dec 16, 2024
9bfeac3
fix test
River2000i Dec 16, 2024
fb654e8
fix test
River2000i Dec 16, 2024
7e92d40
fix test
River2000i Dec 16, 2024
c04aeb9
revert
River2000i Dec 16, 2024
a2b0494
fmt
River2000i Dec 16, 2024
33b326c
fix test
River2000i Dec 16, 2024
eb1a609
fix test
River2000i Dec 16, 2024
faf557d
fix test
River2000i Dec 16, 2024
60ff231
fix test
River2000i Dec 16, 2024
2ecd31b
fix test
River2000i Dec 16, 2024
bb77e93
fix test
River2000i Dec 16, 2024
85acc9b
fix test
River2000i Dec 17, 2024
9298b97
fmt
River2000i Dec 17, 2024
2389c33
fix test
River2000i Dec 17, 2024
b7225d8
fmt
River2000i Dec 17, 2024
96414d9
use tls content
River2000i Dec 17, 2024
008383a
write certificate files
River2000i Dec 18, 2024
fa8fb73
add comment
River2000i Dec 18, 2024
0d1814a
fix test
River2000i Dec 18, 2024
8eb2320
fmt
River2000i Dec 18, 2024
a68e2fb
debug
River2000i Dec 18, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
add ut
  • Loading branch information
River2000i committed Dec 10, 2024
commit ee0bd4c5c569de40282b21f494170b36aa7ca564
7 changes: 1 addition & 6 deletions dm/loader/lightning.go
Original file line number Diff line number Diff line change
Expand Up @@ -332,17 +332,12 @@ func GetLightningConfig(globalCfg *lcfg.GlobalConfig, subtaskCfg *config.SubTask
if err := cfg.LoadFromGlobal(globalCfg); err != nil {
return nil, err
}
if subtaskCfg.To.Security != nil {
cfg.TiDB.Security.CABytes = subtaskCfg.To.Security.SSLCABytes
cfg.TiDB.Security.CertBytes = subtaskCfg.To.Security.SSLCertBytes
cfg.TiDB.Security.KeyBytes = subtaskCfg.To.Security.SSLKeyBytes
}
cfg.TiDB.Security = &globalCfg.Security
if subtaskCfg.LoaderConfig.Security != nil {
cfg.Security.CABytes = subtaskCfg.LoaderConfig.Security.SSLCABytes
cfg.Security.CertBytes = subtaskCfg.LoaderConfig.Security.SSLCertBytes
cfg.Security.KeyBytes = subtaskCfg.LoaderConfig.Security.SSLKeyBytes
}

// TableConcurrency is adjusted to the value of RegionConcurrency
// when using TiDB backend.
// TODO: should we set the TableConcurrency separately.
Expand Down
113 changes: 113 additions & 0 deletions dm/loader/lightning_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,75 @@ import (
"github.com/pingcap/tidb/pkg/lightning/common"
lcfg "github.com/pingcap/tidb/pkg/lightning/config"
"github.com/pingcap/tiflow/dm/config"
"github.com/pingcap/tiflow/dm/config/dbconfig"
"github.com/pingcap/tiflow/dm/config/security"
"github.com/pingcap/tiflow/dm/pkg/terror"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/stretchr/testify/require"
)

var (
caContent = []byte(`-----BEGIN CERTIFICATE-----
MIIBGDCBwAIJAOjYXLFw5V1HMAoGCCqGSM49BAMCMBQxEjAQBgNVBAMMCWxvY2Fs
aG9zdDAgFw0yMDAzMTcxMjAwMzNaGA8yMjkzMTIzMTEyMDAzM1owFDESMBAGA1UE
AwwJbG9jYWxob3N0MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEglCIJD8uVBfD
kuM+UQP+VA7Srbz17WPLA0Sqc+sQ2p6fT6HYKCW60EXiZ/yEC0925iyVbXEEbX4J
xCc2Heow5TAKBggqhkjOPQQDAgNHADBEAiAILL3Zt/3NFeDW9c9UAcJ9lc92E0ZL
GNDuH6i19Fex3wIgT0ZMAKAFSirGGtcLu0emceuk+zVKjJzmYbsLdpj/JuQ=
-----END CERTIFICATE-----
`)
certContent = []byte(`-----BEGIN CERTIFICATE-----
MIIBZDCCAQqgAwIBAgIJAIT/lgXUc1JqMAoGCCqGSM49BAMCMBQxEjAQBgNVBAMM
CWxvY2FsaG9zdDAgFw0yMDAzMTcxMjAwMzNaGA8yMjkzMTIzMTEyMDAzM1owDTEL
MAkGA1UEAwwCZG0wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAASBA6/ltA7vErXq
9laHAmqXPa+XX34BdbZCXspDIaIElVK8tvIMs6uQh4WUc3TiKpDf1IpI5J94ZJ9G
3p2hTohwo0owSDAaBgNVHREEEzARgglsb2NhbGhvc3SHBH8AAAEwCwYDVR0PBAQD
AgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATAKBggqhkjOPQQDAgNI
ADBFAiEAx6ljJ+tNa55ypWLGNqmXlB4UdMmKmE4RSKJ8mmEelfECIG2ZmCE59rv5
wImM6KnK+vM2QnEiISH3PeYyyRzQzycu
-----END CERTIFICATE-----
`)
keyContent = []byte(`-----BEGIN EC PARAMETERS-----
BggqhkjOPQMBBw==
-----END EC PARAMETERS-----
-----BEGIN EC PRIVATE KEY-----
MHcCAQEEICF/GDtVxhTPTP501nOu4jgwGSDY01xN+61xd9MfChw+oAoGCCqGSM49
AwEHoUQDQgAEgQOv5bQO7xK16vZWhwJqlz2vl19+AXW2Ql7KQyGiBJVSvLbyDLOr
kIeFlHN04iqQ39SKSOSfeGSfRt6doU6IcA==
-----END EC PRIVATE KEY-----
`)
caContent2 = []byte(`-----BEGIN CERTIFICATE-----
MIIBGDCBwAIJAOjYXLFw5V1HMAoGCCqGSM49BAMCMBQxEjAQBgNVBAMMCWxvY2Fs
aG9zdDAgFw0yMDAzMTcxMjAwMzNaGA8yMjkzMTIzMTEyMDAzM1owFDESMBAGA1UE
AwwJbG9jYWxob3N0MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEglCIJD8uVBfD
kuM+UQP+VA7Srbz17WPLA0Sqc+sQ2p6fT6HYKCW60EXiZ/yEC0925iyVbXEEbX4J
xCc2Heow5TAKBggqhkjOPQQDAgNHADBEAiAILL3Zt/3NFeDW9c9UAcJ9lc92E0ZL
GNDuH6i19Fex3wIgT0ZMAKAFSirGGtcLu0emceuk+zVKjJzmYbsLdpj/JuQ=
-----END CERTIFICATE-----
`)
certContent2 = []byte(`-----BEGIN CERTIFICATE-----
MIIBcDCCARWgAwIBAgIUNC83r8QT87G4uCeW2wUMzaDbCvAwCgYIKoZIzj0EAwIw
FDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTI0MTIwNjAzNDgxMloXDTM0MTIwNDAz
NDgxMlowDzENMAsGA1UEAwwEdGlkYjBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IA
BOWs95/gIDUG116NoBZhABn6uWbSIvDva3mwsHnw9PGevSb23Q9t1kl7y1dQpMpT
lSQ/31FOIgCul/RTMYre95CjSjBIMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAA
ATALBgNVHQ8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMBMAoG
CCqGSM49BAMCA0kAMEYCIQDDPgmo3olaw1D/7YW3463jvuSBd4w2Z3Ai/BHgZB7d
BAIhALKIhAqB1ffI5XdSdfnznqfwX6FY9c9POlJNfkghB07e
-----END CERTIFICATE-----
`)
keyContent2 = []byte(`-----BEGIN EC PARAMETERS-----
BggqhkjOPQMBBw==
-----END EC PARAMETERS-----
-----BEGIN EC PRIVATE KEY-----
MHcCAQEEIMdUrYsjfC9TNSMKAcGWYB9hmKKzyxuxMfRwDGkc03PzoAoGCCqGSM49
AwEHoUQDQgAE5az3n+AgNQbXXo2gFmEAGfq5ZtIi8O9rebCwefD08Z69JvbdD23W
SXvLV1CkylOVJD/fUU4iAK6X9FMxit73kA==
-----END EC PRIVATE KEY-----
`)
)

func TestSetLightningConfig(t *testing.T) {
t.Parallel()

Expand Down Expand Up @@ -99,6 +162,56 @@ func TestGetLightiningConfig(t *testing.T) {
})
require.NoError(t, err)
require.Equal(t, lcfg.CheckpointDriverMySQL, conf.Checkpoint.Driver)

cases := []struct {
globalSecurityCfg *lcfg.Security
loaderSecurityCfg *security.Security
toSecurityCfg *security.Security
}{
{
globalSecurityCfg: &lcfg.Security{CABytes: caContent, CertBytes: certContent, KeyBytes: keyContent},
loaderSecurityCfg: &security.Security{SSLCABytes: caContent2, SSLCertBytes: certContent2, SSLKeyBytes: keyContent2},
toSecurityCfg: &security.Security{SSLCABytes: caContent, SSLCertBytes: certContent, SSLKeyBytes: keyContent},
},
{
globalSecurityCfg: &lcfg.Security{CABytes: caContent},
loaderSecurityCfg: &security.Security{SSLCABytes: caContent2, SSLCertBytes: certContent2, SSLKeyBytes: keyContent2},
toSecurityCfg: &security.Security{SSLCABytes: caContent},
},
{
globalSecurityCfg: &lcfg.Security{CABytes: caContent, CertBytes: certContent, KeyBytes: keyContent},
toSecurityCfg: &security.Security{SSLCABytes: caContent, SSLCertBytes: certContent, SSLKeyBytes: keyContent},
},
{
globalSecurityCfg: &lcfg.Security{CABytes: caContent},
toSecurityCfg: &security.Security{SSLCABytes: caContent},
},
{
globalSecurityCfg: &lcfg.Security{},
toSecurityCfg: &security.Security{},
},
}
for _, c := range cases {
conf, err = GetLightningConfig(
&lcfg.GlobalConfig{Security: *c.globalSecurityCfg},
&config.SubTaskConfig{
LoaderConfig: config.LoaderConfig{Security: c.loaderSecurityCfg},
To: dbconfig.DBConfig{Security: c.toSecurityCfg},
})
require.NoError(t, err)
require.Equal(t, c.globalSecurityCfg.CABytes, conf.TiDB.Security.CABytes)
require.Equal(t, c.globalSecurityCfg.CertBytes, conf.TiDB.Security.CertBytes)
require.Equal(t, c.globalSecurityCfg.KeyBytes, conf.TiDB.Security.KeyBytes)
if c.loaderSecurityCfg == nil {
require.Equal(t, c.globalSecurityCfg.CABytes, conf.Security.CABytes)
require.Equal(t, c.globalSecurityCfg.CertBytes, conf.Security.CertBytes)
require.Equal(t, c.globalSecurityCfg.KeyBytes, conf.Security.KeyBytes)
} else {
require.Equal(t, c.loaderSecurityCfg.SSLCABytes, conf.Security.CABytes)
require.Equal(t, c.loaderSecurityCfg.SSLCertBytes, conf.Security.CertBytes)
require.Equal(t, c.loaderSecurityCfg.SSLKeyBytes, conf.Security.KeyBytes)
}
}
}

func TestMetricProxies(t *testing.T) {
Expand Down
Loading