Merge #149517 #150847 #151122

149517: build: add crdb_bench flag r=herkolategan a=herkolategan

A recent change (#148576) causes some test initializations to throw a panic if
the binary was not built with `crdb_test`.

```
./dev bench pkg/sql/catalog/lease -f=BenchmarkAcquireLeaseConcurrent
$ bazel test pkg/sql/catalog/lease:all --nocache_test_results --test_arg -test.run=- --test_arg -test.bench=BenchmarkAcquireLeaseConcurrent --test_sharding_strategy=disabled --test_arg -test.cpu --test_arg 1 --test_arg -test.benchmem --crdb_test_off --test_env COCKROACH_TEST_FIXTURES_DIR=/Users/herko/Library/Caches/crdb-test-fixtures --sandbox_writable_path=/Users/herko/Library/Caches/crdb-test-fixtures --test_output streamed

panic: Testing override for schema_locked used in non-test binary.

goroutine 1 [running]:
github.com/cockroachdb/cockroach/pkg/sql.TestForceDisableCreateTableWithSchemaLocked(...)
	pkg/sql/exec_util.go:773
github.com/cockroachdb/cockroach/pkg/sql/catalog/lease_test.TestMain(0x1400132e620?)
	pkg/sql/catalog/lease/main_test.go:29 +0xcc
main.main()
```

This is problematic for benchmarks, which are not built with `crdb_test`.
Benchmarks are not built with the `crdb_test` flag, because this flag enables
metamorphic variables and extra assertions which could interfere with
performance testing.

This change adds a new `crdb_bench` flag, which is disabled by default. The
assertions added in #148576 are now also skipped when `crdb_bench` is enabled.

Informs: #148576
Fixes: #149339

Epic: None
Release note: None

150847: asim: introduce eval configurations and a chart viewer r=tbg a=tbg

This PR

- replaces the manual plot command with the automatic generation of all relevant charts.
- moves from static PNG image outputs to a single JSON file per run, which can be viewed and analyzed in an interactive HTML viewer.
- excludes those generated files from the repository. This significantly reduces churn.
- introduces eval configurations (e.g., sma, mma, both) that automatically set the appropriate cluster settings for a test run.
- consolidates numerous redundant and messy test files into fewer, cleaner tests that leverage the new configuration system.
- add validation to warn against unrealistic test parameters.
- address all such warnings.

I think some more work is needed to tell, based on the datadriven files along, whether the configured allocator is "doing a good enough job". But at least there are fewer test scenarios now, and none of them configure obviously bogus loads or capacities.

Epic: CRDB-52631

151122: kvserver: dump memFS when TestCheckConsistencyInconsistent fails r=tbg a=tbg

Informs #150660.

Epic: none

Co-authored-by: Herko Lategan <herko@cockroachlabs.com>
Co-authored-by: Tobias Grieger <tobias.b.grieger@gmail.com>

Author: 3 people

.bazelrc

@@ -22,6 +22,7 @@ startup --host_jvm_args=-DBAZEL_TRACK_SOURCE_DIRECTORIES=1
 build --flag_alias=bazel_code_coverage=//build/toolchains:bazel_code_coverage_flag
 build --flag_alias=crdb_test=//build/toolchains:crdb_test_flag
 build --flag_alias=crdb_test_off=//build/toolchains:crdb_test_off_flag
+build --flag_alias=crdb_bench=//build/toolchains:crdb_bench_flag
 build --flag_alias=cross=//build/toolchains:cross_flag
 build --flag_alias=dev=//build/toolchains:dev_flag
 build --flag_alias=force_build_cdeps=//build/toolchains:force_build_cdeps_flag

build/teamcity/cockroach/nightlies/microbenchmark_build_support.sh

@@ -24,7 +24,13 @@ function build_and_upload_binaries() {
     return
   fi
 
-  config_args="--config=crosslinux --crdb_test_off"
+  # Check if crdb_bench flag is supported, since we could be building an older
+  # version that does not support it.
+  if grep -q "crdb_bench" .bazelrc; then
+    config_args="--config=crosslinux --crdb_test_off --crdb_bench"
+  else
+    config_args="--config=crosslinux --crdb_test_off"
+  fi
   bazel clean
   go_test_targets=$(bazel query kind\(go_test, //$BENCH_PACKAGE:all\))
   bazel build $config_args $go_test_targets

build/toolchains/BUILD.bazel

@@ -299,6 +299,19 @@ config_setting(
     },
 )
 
+bool_flag(
+    name = "crdb_bench_flag",
+    build_setting_default = False,
+    visibility = ["//visibility:public"],
+)
+
+config_setting(
+    name = "crdb_bench",
+    flag_values = {
+        ":crdb_bench_flag": "true",
+    },
+)
+
 bool_flag(
     name = "dev_flag",
     build_setting_default = False,

dev

@@ -8,7 +8,7 @@ fi
 set -euo pipefail
 
 # Bump this counter to force rebuilding `dev` on all machines.
-DEV_VERSION=111
+DEV_VERSION=112
 
 THIS_DIR=$(cd "$(dirname "$0")" && pwd)
 BINARY_DIR=$THIS_DIR/bin/dev-versions

go.mod

@@ -244,7 +244,6 @@ require (
 	golang.org/x/perf v0.0.0-20230113213139-801c7ef9e5c5
 	golang.org/x/term v0.30.0
 	gonum.org/v1/gonum v0.15.1
-	gonum.org/v1/plot v0.14.0
 	google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9
 	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
@@ -458,6 +457,7 @@ require (
 	golang.org/x/image v0.21.0 // indirect
 	golang.org/x/tools/go/vcs v0.1.0-deprecated // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
+	gonum.org/v1/plot v0.14.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto/googleapis/bytestream v0.0.0-20230525234009-2805bf891e89 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect

pkg/BUILD.bazel

@@ -819,6 +819,7 @@ ALL_TESTS = [
     "//pkg/util/unique:unique_test",
     "//pkg/util/uuid:uuid_test",
     "//pkg/util/vector:vector_test",
+    "//pkg/util/vfsutil:vfsutil_test",
     "//pkg/util:util_test",
     "//pkg/workload/bank:bank_test",
     "//pkg/workload/cli:cli_test",
@@ -2792,6 +2793,8 @@ GO_TARGETS = [
     "//pkg/util/uuid:uuid_test",
     "//pkg/util/vector:vector",
     "//pkg/util/vector:vector_test",
+    "//pkg/util/vfsutil:vfsutil",
+    "//pkg/util/vfsutil:vfsutil_test",
     "//pkg/util:util",
     "//pkg/util:util_test",
     "//pkg/workload/bank:bank",

pkg/cmd/dev/bench.go

@@ -140,7 +140,10 @@ func (d *dev) bench(cmd *cobra.Command, commandLine []string) error {
 	if runSepProcessTenant {
 		args = append(args, "--test_arg", "-run-sep-process-tenant")
 	}
-	args = append(args, "--crdb_test_off")
+	// The `crdb_test` flag enables metamorphic variables and various "extra" debug
+	// checking code paths that can interfere with performance testing, hence
+	// it should disabled for benchmarks.
+	args = append(args, "--crdb_test_off", "--crdb_bench")
 	if testArgs != "" {
 		goTestArgs, err := d.getGoTestArgs(ctx, testArgs)
 		if err != nil {

pkg/cmd/dev/testdata/datadriven/bench

@@ -2,35 +2,35 @@ exec
 dev bench pkg/spanconfig/...
 ----
 echo $HOME/.cache
-bazel test pkg/spanconfig/...:all --nocache_test_results --test_arg -test.run=- --test_arg -test.bench=. --test_sharding_strategy=disabled --test_arg -test.cpu --test_arg 1 --test_arg -test.benchmem --crdb_test_off --test_env COCKROACH_TEST_FIXTURES_DIR=crdb-mock-test-fixtures/crdb-test-fixtures --sandbox_writable_path=crdb-mock-test-fixtures/crdb-test-fixtures --test_output streamed
+bazel test pkg/spanconfig/...:all --nocache_test_results --test_arg -test.run=- --test_arg -test.bench=. --test_sharding_strategy=disabled --test_arg -test.cpu --test_arg 1 --test_arg -test.benchmem --crdb_test_off --crdb_bench --test_env COCKROACH_TEST_FIXTURES_DIR=crdb-mock-test-fixtures/crdb-test-fixtures --sandbox_writable_path=crdb-mock-test-fixtures/crdb-test-fixtures --test_output streamed
 
 exec
 dev bench pkg/sql/parser --filter=BenchmarkParse
 ----
 echo $HOME/.cache
-bazel test pkg/sql/parser:all --nocache_test_results --test_arg -test.run=- --test_arg -test.bench=BenchmarkParse --test_sharding_strategy=disabled --test_arg -test.cpu --test_arg 1 --test_arg -test.benchmem --crdb_test_off --test_env COCKROACH_TEST_FIXTURES_DIR=crdb-mock-test-fixtures/crdb-test-fixtures --sandbox_writable_path=crdb-mock-test-fixtures/crdb-test-fixtures --test_output streamed
+bazel test pkg/sql/parser:all --nocache_test_results --test_arg -test.run=- --test_arg -test.bench=BenchmarkParse --test_sharding_strategy=disabled --test_arg -test.cpu --test_arg 1 --test_arg -test.benchmem --crdb_test_off --crdb_bench --test_env COCKROACH_TEST_FIXTURES_DIR=crdb-mock-test-fixtures/crdb-test-fixtures --sandbox_writable_path=crdb-mock-test-fixtures/crdb-test-fixtures --test_output streamed
 
 exec
 dev bench pkg/sql/parser --filter=BenchmarkParse --stream-output=false
 ----
 echo $HOME/.cache
-bazel test pkg/sql/parser:all --nocache_test_results --test_arg -test.run=- --test_arg -test.bench=BenchmarkParse --test_sharding_strategy=disabled --test_arg -test.cpu --test_arg 1 --test_arg -test.benchmem --crdb_test_off --test_env COCKROACH_TEST_FIXTURES_DIR=crdb-mock-test-fixtures/crdb-test-fixtures --sandbox_writable_path=crdb-mock-test-fixtures/crdb-test-fixtures --test_output errors
+bazel test pkg/sql/parser:all --nocache_test_results --test_arg -test.run=- --test_arg -test.bench=BenchmarkParse --test_sharding_strategy=disabled --test_arg -test.cpu --test_arg 1 --test_arg -test.benchmem --crdb_test_off --crdb_bench --test_env COCKROACH_TEST_FIXTURES_DIR=crdb-mock-test-fixtures/crdb-test-fixtures --sandbox_writable_path=crdb-mock-test-fixtures/crdb-test-fixtures --test_output errors
 
 exec
 dev bench pkg/bench -f=BenchmarkTracing/1node/scan/trace=off --count=2 --bench-time=10x --bench-mem=false
 ----
 echo $HOME/.cache
-bazel test pkg/bench:all --nocache_test_results --test_arg -test.run=- --test_arg -test.bench=BenchmarkTracing/1node/scan/trace=off --test_sharding_strategy=disabled --test_arg -test.cpu --test_arg 1 --test_arg -test.count=2 --test_arg -test.benchtime=10x --crdb_test_off --test_env COCKROACH_TEST_FIXTURES_DIR=crdb-mock-test-fixtures/crdb-test-fixtures --sandbox_writable_path=crdb-mock-test-fixtures/crdb-test-fixtures --test_output streamed
+bazel test pkg/bench:all --nocache_test_results --test_arg -test.run=- --test_arg -test.bench=BenchmarkTracing/1node/scan/trace=off --test_sharding_strategy=disabled --test_arg -test.cpu --test_arg 1 --test_arg -test.count=2 --test_arg -test.benchtime=10x --crdb_test_off --crdb_bench --test_env COCKROACH_TEST_FIXTURES_DIR=crdb-mock-test-fixtures/crdb-test-fixtures --sandbox_writable_path=crdb-mock-test-fixtures/crdb-test-fixtures --test_output streamed
 
 exec
 dev bench pkg/spanconfig/spanconfigkvsubscriber -f=BenchmarkSpanConfigDecoder --cpus=10 --ignore-cache=false -v --timeout=50s
 ----
 echo $HOME/.cache
-bazel test --local_cpu_resources=10 --test_timeout=50 pkg/spanconfig/spanconfigkvsubscriber:all --test_arg -test.run=- --test_arg -test.bench=BenchmarkSpanConfigDecoder --test_sharding_strategy=disabled --test_arg -test.cpu --test_arg 1 --test_arg -test.v --test_arg -test.benchmem --crdb_test_off --test_env COCKROACH_TEST_FIXTURES_DIR=crdb-mock-test-fixtures/crdb-test-fixtures --sandbox_writable_path=crdb-mock-test-fixtures/crdb-test-fixtures --test_output streamed
+bazel test --local_cpu_resources=10 --test_timeout=50 pkg/spanconfig/spanconfigkvsubscriber:all --test_arg -test.run=- --test_arg -test.bench=BenchmarkSpanConfigDecoder --test_sharding_strategy=disabled --test_arg -test.cpu --test_arg 1 --test_arg -test.v --test_arg -test.benchmem --crdb_test_off --crdb_bench --test_env COCKROACH_TEST_FIXTURES_DIR=crdb-mock-test-fixtures/crdb-test-fixtures --sandbox_writable_path=crdb-mock-test-fixtures/crdb-test-fixtures --test_output streamed
 
 exec
 dev bench pkg/bench -f='BenchmarkTracing/1node/scan/trace=off' --test-args '-test.memprofile=mem.out -test.cpuprofile=cpu.out'
 ----
 bazel info workspace --color=no
 echo $HOME/.cache
-bazel test pkg/bench:all --nocache_test_results --test_arg -test.run=- --test_arg -test.bench=BenchmarkTracing/1node/scan/trace=off --test_sharding_strategy=disabled --test_arg -test.cpu --test_arg 1 --test_arg -test.benchmem --crdb_test_off --test_arg -test.outputdir=crdb-checkout --sandbox_writable_path=crdb-checkout --test_arg -test.memprofile=mem.out --test_arg -test.cpuprofile=cpu.out --test_env COCKROACH_TEST_FIXTURES_DIR=crdb-mock-test-fixtures/crdb-test-fixtures --sandbox_writable_path=crdb-mock-test-fixtures/crdb-test-fixtures --test_output streamed
+bazel test pkg/bench:all --nocache_test_results --test_arg -test.run=- --test_arg -test.bench=BenchmarkTracing/1node/scan/trace=off --test_sharding_strategy=disabled --test_arg -test.cpu --test_arg 1 --test_arg -test.benchmem --crdb_test_off --crdb_bench --test_arg -test.outputdir=crdb-checkout --sandbox_writable_path=crdb-checkout --test_arg -test.memprofile=mem.out --test_arg -test.cpuprofile=cpu.out --test_env COCKROACH_TEST_FIXTURES_DIR=crdb-mock-test-fixtures/crdb-test-fixtures --sandbox_writable_path=crdb-mock-test-fixtures/crdb-test-fixtures --test_output streamed

pkg/kv/kvserver/BUILD.bazel

@@ -562,6 +562,7 @@ go_test(
         "//pkg/util/tracing/tracingpb",
         "//pkg/util/uint128",
         "//pkg/util/uuid",
+        "//pkg/util/vfsutil",
         "@com_github_cockroachdb_cockroach_go_v2//crdb",
         "@com_github_cockroachdb_crlib//crtime",
         "@com_github_cockroachdb_datadriven//:datadriven",

pkg/kv/kvserver/asim/mmaintegration/mma_store_rebalancer.go

@@ -88,7 +88,7 @@ func NewMMAStoreRebalancer(
 // Tick is called periodically to check for and apply rebalancing operations
 // using mmaprototype.Allocator.
 func (msr *MMAStoreRebalancer) Tick(ctx context.Context, tick time.Time, s state.State) {
-	ctx = msr.ResetAndAnnotateCtx(ctx)
+	ctx = msr.AnnotateCtx(ctx)
 	ctx = logtags.AddTag(ctx, "t", tick.Sub(msr.settings.StartTime))
 
 	if !msr.currentlyRebalancing &&