load data: physical mode part1 (#42817)

ref #40499
pingcap · Apr 7, 2023 · 2af85d1 · 2af85d1
1 parent bd3a036
commit 2af85d1
Show file tree

Hide file tree

Showing 11 changed files with 1,499 additions and 20 deletions.
diff --git a/br/pkg/lightning/mydump/loader.go b/br/pkg/lightning/mydump/loader.go
@@ -69,13 +69,14 @@ func (m *MDDatabaseMeta) GetSchema(ctx context.Context, store storage.ExternalSt
 
 // MDTableMeta contains some parsed metadata for a table in the source by MyDumper Loader.
 type MDTableMeta struct {
-	DB           string
-	Name         string
-	SchemaFile   FileInfo
-	DataFiles    []FileInfo
-	charSet      string
-	TotalSize    int64
-	IndexRatio   float64
+	DB         string
+	Name       string
+	SchemaFile FileInfo
+	DataFiles  []FileInfo
+	charSet    string
+	TotalSize  int64
+	IndexRatio float64
+	// default to true, and if we do precheck, this var is updated using data sampling result, so it's not accurate.
 	IsRowOrdered bool
 }
 

diff --git a/executor/importer/BUILD.bazel b/executor/importer/BUILD.bazel
@@ -2,44 +2,70 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "importer",
-    srcs = ["import.go"],
+    srcs = [
+        "chunk_process.go",
+        "engine_process.go",
+        "import.go",
+        "kv_encode.go",
+        "table_import.go",
+    ],
     importpath = "github.com/pingcap/tidb/executor/importer",
     visibility = ["//visibility:public"],
     deps = [
+        "//br/pkg/lightning/backend",
+        "//br/pkg/lightning/backend/encode",
+        "//br/pkg/lightning/backend/kv",
+        "//br/pkg/lightning/backend/local",
+        "//br/pkg/lightning/checkpoints",
         "//br/pkg/lightning/common",
         "//br/pkg/lightning/config",
         "//br/pkg/lightning/log",
         "//br/pkg/lightning/mydump",
+        "//br/pkg/lightning/verification",
         "//br/pkg/storage",
+        "//config",
+        "//expression",
         "//kv",
+        "//meta/autoid",
         "//parser/ast",
         "//parser/mysql",
         "//parser/terror",
         "//planner/core",
         "//sessionctx",
         "//sessionctx/variable",
         "//table",
+        "//table/tables",
+        "//tablecodec",
+        "//types",
+        "//util",
         "//util/chunk",
         "//util/dbterror",
         "//util/dbterror/exeerrors",
+        "//util/filter",
         "//util/intest",
         "//util/logutil",
         "//util/stringutil",
+        "@com_github_docker_go_units//:go-units",
         "@com_github_pingcap_errors//:errors",
         "@com_github_pingcap_log//:log",
         "@com_github_tikv_client_go_v2//config",
+        "@com_github_tikv_client_go_v2//tikv",
+        "@org_uber_go_multierr//:multierr",
         "@org_uber_go_zap//:zap",
     ],
 )
 
 go_test(
     name = "importer_test",
     timeout = "short",
-    srcs = ["import_test.go"],
+    srcs = [
+        "engine_process_test.go",
+        "import_test.go",
+    ],
     embed = [":importer"],
     flaky = True,
     race = "on",
-    shard_count = 4,
+    shard_count = 5,
     deps = [
         "//br/pkg/errors",
         "//br/pkg/lightning/config",
@@ -50,6 +76,7 @@ go_test(
         "//util/dbterror/exeerrors",
         "//util/mock",
         "@com_github_pingcap_errors//:errors",
+        "@com_github_pingcap_log//:log",
         "@com_github_stretchr_testify//require",
     ],
 )