DEBUG: diagnose flaky memory-pressure RRCC test on Windows

.bazelci/presubmit.yml

@@ -1,313 +1,26 @@
 ---
-check_docs: 1
+# DEBUG ONLY: trimmed down to a single Windows task that runs
+# testMemoryPressureRestartDuringCachedFetch repeatedly. Do not merge.
 tasks:
-  rockylinux8:
-    shell_commands:
-      - rm -rf $HOME/bazeltest
-      - mkdir $HOME/bazeltest
-    build_flags:
-      - "--config=ci-linux"
-    build_targets:
-      - "//:bazel-distfile.zip"
-      - "//scripts/packages/debian:bazel-debian.deb"
-      - "//scripts/packages:with-jdk/install.sh"
-      - "//src:bazel"
-      - "//src:bazel_jdk_minimal"
-      - "//src:test_repos"
-      - "//..."
-    test_flags:
-      - "--config=ci-linux"
-      - "--test_tag_filters=-no_presubmit"
-    test_targets:
-      - "//src:embedded_tools_size_test"
-      - "//src/test/py/bazel:bazel_module_test"
-      - "//src/test/py/bazel:mod_command_test"
-      - "//src/test/shell/bazel:bazel_example_test"
-      - "//src/test/shell/bazel:bazel_java_test"
-      - "//src/test/shell/bazel:cc_integration_test"
-      - "//src/test/shell/bazel:external_integration_test"
-      - "//src/test/shell/bazel:help_test"
-      - "//src/test/shell/integration:bazel_query_test"
-    include_json_profile:
-      - build
-      - test
-  rockylinux8_arm64:
-    shell_commands:
-      - rm -rf $HOME/bazeltest
-      - mkdir $HOME/bazeltest
-    build_flags:
-      - "--config=ci-linux"
-    build_targets:
-      - "//:bazel-distfile.zip"
-      - "//scripts/packages/debian:bazel-debian.deb"
-      - "//scripts/packages:with-jdk/install.sh"
-      - "//src:bazel"
-      - "//src:bazel_jdk_minimal"
-      - "//src:test_repos"
-      - "//..."
-    include_json_profile:
-      - build
-  fedora39:
-    shell_commands:
-      - rm -rf $HOME/bazeltest
-      - mkdir $HOME/bazeltest
-    build_flags:
-      - "--config=ci-linux"
-    build_targets:
-      - "//src:bazel"
-      - "//src:bazel_jdk_minimal"
-      - "//src/main/java/..."
-    include_json_profile:
-      - build
-      - test
-  ubuntu2404:
-    shards: 8
-    shell_commands:
-      - rm -rf $HOME/bazeltest
-      - mkdir $HOME/bazeltest
-    build_flags:
-      - "--config=ci-linux"
-    build_targets:
-      - "//src:bazel"
-      - "//src:bazel_jdk_minimal"
-      - "//src:test_repos"
-      - "//..."
-    test_flags:
-      - "--config=ci-linux"
-      - "--test_tag_filters=-no_presubmit"
-    test_targets:
-      - "//..."
-    include_json_profile:
-      - build
-      - test
-  ubuntu2404_clang:
-    platform: ubuntu2404
-    environment:
-      CC: clang
-      CC_CONFIGURE_DEBUG: 1
-    name: "Clang"
-    shell_commands:
-      - rm -rf $HOME/bazeltest
-      - mkdir $HOME/bazeltest
-    build_flags:
-      - "--config=ci-linux"
-    build_targets:
-      - "//src:bazel"
-      - "//src:bazel_jdk_minimal"
-      - "//src:test_repos"
-      - "//..."
-    test_flags:
-      - "--config=ci-linux"
-    test_targets:
-      - "//src/test/shell/bazel:cc_integration_test"
-    include_json_profile:
-      - build
-      - test
-  ubuntu2204:
-    shell_commands:
-      - rm -rf $HOME/bazeltest
-      - mkdir $HOME/bazeltest
-    build_flags:
-      - "--config=ci-linux"
-    build_targets:
-      - "//src:bazel"
-      - "//src:bazel_jdk_minimal"
-      - "//src:test_repos"
-      - "//..."
-    test_flags:
-      - "--config=ci-linux"
-      - "--test_tag_filters=-no_presubmit"
-    test_targets:
-      - "//src:embedded_tools_size_test"
-      - "//src/test/py/bazel:bazel_module_test"
-      - "//src/test/py/bazel:mod_command_test"
-      - "//src/test/shell/bazel:bazel_example_test"
-      - "//src/test/shell/bazel:bazel_java_test"
-      - "//src/test/shell/bazel:cc_integration_test"
-      - "//src/test/shell/bazel:external_integration_test"
-      - "//src/test/shell/bazel:help_test"
-      - "//src/test/shell/integration:bazel_query_test"
-    include_json_profile:
-      - build
-      - test
-  macos:
-    environment:
-      ANDROID_HOME: ""
-      ANDROID_NDK_HOME: ""
-    shell_commands:
-      - rm -rf $HOME/bazeltest
-      - mkdir $HOME/bazeltest
-      - ln -sf $OUTPUT_BASE/external $HOME/bazeltest/external
-    build_flags:
-      - "--config=ci-macos"
-    build_targets:
-      - "//src:bazel"
-      - "//src:bazel_jdk_minimal"
-      - "//src:java_tools_zip"
-      - "//src:java_tools_prebuilt_zip"
-    include_json_profile:
-      - build
-  macos_arm64:
-    shards: 6
-    shell_commands:
-      - rm -rf $HOME/bazeltest
-      - mkdir $HOME/bazeltest
-      - ln -sf $OUTPUT_BASE/external $HOME/bazeltest/external
-    build_flags:
-      - "--config=ci-macos"
-    build_targets:
-      - "//src:bazel"
-      - "//src:bazel_jdk_minimal"
-      - "//src:test_repos"
-      - "//src:java_tools_zip"
-      - "//src:java_tools_prebuilt_zip"
-    test_flags:
-      - "--config=ci-macos"
-      - "--test_tag_filters=-no_presubmit,-no_macos"
-      # Fine tune the number of test jobs running in parallel to avoid timeout
-      - "--local_test_jobs=2"
-      # Increase the test timeout by 20% to avoid flaky test failures due to timeout
-      - "--test_timeout=72,360,1080,4320"
-    test_targets:
-      - "//src:embedded_tools_size_test"
-      - "//src/test/java/com/google/devtools/build/lib/rules/apple:AppleRulesTests"
-      - "//src/test/java/com/google/devtools/build/lib/rules/objc:ObjcRulesTests"
-      - "//src/test/java/com/google/devtools/build/lib/skyframe:MacOSXFsEventsDiffAwarenessTest"
-      - "//src/test/py/bazel:bazel_module_test"
-      - "//src/test/py/bazel:mod_command_test"
-      - "//src/test/shell/bazel:bazel_example_test"
-      - "//src/test/shell/bazel:bazel_java_test"
-      - "//src/test/shell/bazel:cc_integration_test"
-      - "//src/test/shell/bazel:cpp_darwin_integration_test"
-      - "//src/test/shell/bazel:external_integration_test"
-      - "//src/test/shell/bazel:help_test"
-      - "//src/test/shell/bazel/apple:bazel_apple_test"
-      - "//src/test/shell/bazel/apple:bazel_objc_test"
-      - "//src/test/shell/integration:bazel_query_test"
-    include_json_profile:
-      - build
-      - test
   windows:
-    shards: 10
     setup:
       - mkdir C:\b
       - mklink /J C:\b\bazeltest_external %OUTPUT_BASE:/=\%\external
     build_flags:
       - "--config=ci-windows"
     build_targets:
       - "//src:bazel.exe"
-      - "//src:bazel_jdk_minimal"
-      - "//src:test_repos"
-      - "//src/main/java/..."
-      - "//src:java_tools_zip"
-      - "//src:java_tools_prebuilt_zip"
     test_flags:
       - "--config=ci-windows"
       # We need to keep --test_tag_filters here for the bazel-diff integration.
       - "--test_tag_filters=-no_presubmit,-no_windows,-slow"
+      - "--test_filter=RemoteRepoContentsCacheTest.testMemoryPressureRestartDuringCachedFetch"
+      - "--test_sharding_strategy=disabled"
+      - "--runs_per_test=6"
+      - "--local_test_jobs=2"
+      - "--nocache_test_results"
+      - "--flaky_test_attempts=1"
+      - "--test_output=all"
+      - "--test_timeout=3600"
     test_targets:
-      - "//src:embedded_tools_size_test"
-      - "//src/test/cpp/..."
-      - "//src/test/java/com/google/devtools/build/lib/..."
-      - "//src/test/java/com/google/devtools/build/skyframe/..."
-      - "//src/test/java/com/google/devtools/common/options/..."
-      - "//src/test/native/windows/..."
-      - "//src/test/py/bazel/..."
-      - "//src/test/shell/..."
-      - "//src/tools/launcher/..."
-      - "//src/tools/one_version/..."
-      - "//src/tools/singlejar/..."
-      - "//third_party/def_parser/..."
-      - "//tools/aquery_differ/..."
-      - "//tools/bash/..."
-      - "//tools/build_defs/..."
-      - "//tools/java/..."
-      - "//tools/jdk/..."
-      - "//tools/python/..."
-      - "//tools/test/..."
-      # Re-enable the following tests on Windows:
-      # https://github.com/bazelbuild/bazel/issues/4292
-      - "-//src/test/shell/bazel/remote/..."
-      - "-//tools/python:pywrapper_test"
-    include_json_profile:
-      - build
-      - test
-  windows_arm64:
-    platform: windows
-    name: "Windows (arm64)"
-    setup:
-      - mkdir C:\b
-      - mklink /J C:\b\bazeltest_external %OUTPUT_BASE:/=\%\external
-    build_flags:
-      - "--config=ci-windows"
-      - "--config=windows_arm64"
-    build_targets:
-      - "//src:bazel.exe"
-      - "//src:bazel_nojdk.exe"
-    include_json_profile:
-      - build
-  rbe_ubuntu2404:
-    platform: ubuntu2404
-    name: "RBE"
-    build_flags:
-      - "--config=remote"
-      - "--config=ci-common"
-      - "--jobs=200"
-      - "--experimental_output_paths=strip"
-    build_targets:
-      - "//src:bazel"
-      - "//src:bazel_jdk_minimal"
-      - "//..."
-    test_flags:
-      - "--config=remote"
-      - "--config=ci-common"
-      - "--test_tag_filters=-no_presubmit"
-      - "--jobs=200"
-      - "--experimental_output_paths=strip"
-    test_targets:
-      - "//scripts/..."
-      - "//src/java_tools/..."
-      - "//src/test/..."
-      - "//src/tools/execlog/..."
-      - "//src/tools/one_version/..."
-      - "//src/tools/singlejar/..."
-      - "//src/tools/workspacelog/..."
-      - "//third_party/ijar/..."
-      - "//tools/aquery_differ/..."
-      - "//tools/python/..."
-      - "//tools/bash/..."
-      - "//tools/test/..."
-      # See https://github.com/bazelbuild/bazel/issues/8033
-      - "-//src/tools/singlejar:output_jar_simple_test"
-      - "-//src/test/shell/bazel:external_integration_test"
-      - "-//src/test/shell/bazel:bazel_repository_cache_test"
-      - "-//src/test/shell/integration:java_integration_test"
-      - "-//src/test/java/com/google/devtools/build/lib/sandbox/..."
-      # We hit connection timeout error when downloading multiple URLs on RBE, see b/217865760
-      - "-//src/test/py/bazel:bazel_module_test"
-      - "-//src/test/py/bazel:bazel_lockfile_test"
-      - "-//src/test/py/bazel:bazel_overrides_test"
-      - "-//src/test/py/bazel:bazel_repo_mapping_test"
-      - "-//src/test/py/bazel:bazel_tools_proto_test"
-      - "-//src/test/py/bazel:bazel_yanked_versions_test"
-      - "-//src/test/py/bazel:bzlmod_query_test"
-      - "-//src/test/py/bazel:mod_command_test"
-      - "-//src/test/shell/bazel:starlark_repository_test"
-      # Flaky on rbe_ubuntu2004
-      # https://github.com/bazelbuild/continuous-integration/issues/1631
-      - "-//src/test/shell/bazel:bazel_sandboxing_networking_test"
-      # TODO(pcloudy): Re-enable once https://github.com/bazelbuild/bazel/issues/27982 is fixed.
-      - "-//src/test/shell/bazel:srcs_test"
-      # Disable this test since it takes too long on RBE.
-      - "-//src/test/shell/bazel:bazel_proto_library_test"
-    include_json_profile:
-      - build
-      - test
-  docs_ubuntu2404:
-    platform: ubuntu2404
-    name: "Docs"
-    build_flags:
-      - "--config=docs"
-    build_targets:
-      - "//scripts/docs:gen_release_docs"
-      - "//src/main/java/com/google/devtools/build/lib:gen_reference_docs"
+      - "//src/test/py/bazel:remote_repo_contents_cache_test"

src/main/java/com/google/devtools/build/lib/bazel/repository/BUILD

@@ -88,6 +88,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/skyframe",
         "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",
         "//src/main/java/net/starlark/java/eval",
+        "//third_party:flogger",
         "//third_party:guava",
         "//third_party:jsr305",
     ],

src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryFetchFunction.java

@@ -21,6 +21,7 @@
 import com.google.common.base.Throwables;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.flogger.GoogleLogger;
 import com.google.devtools.build.lib.actions.FileStateValue;
 import com.google.devtools.build.lib.actions.FileValue;
 import com.google.devtools.build.lib.analysis.BlazeDirectories;
@@ -89,6 +90,10 @@
 /** A {@link SkyFunction} that fetches the given repository. */
 public final class RepositoryFetchFunction implements SkyFunction {
 
+  // DEBUG ONLY: ground truth for testMemoryPressureRestartDuringCachedFetch, which can otherwise
+  // only observe restarts through the (transient) progress bar.
+  private static final GoogleLogger logger = GoogleLogger.forEnclosingClass();
+
   private final BlazeDirectories directories;
   private final LocalRepoContentsCache repoContentsCache;
   private final Supplier<ImmutableMap<String, String>> repoEnvSupplier;
@@ -196,6 +201,9 @@ public SkyValue compute(SkyKey skyKey, Environment env)
           // This can only happen if the state object was invalidated due to memory pressure, in
           // which case we can simply reattempt the fetch. Show a message and continue into the next
           // `while` iteration.
+          logger.atInfo().log(
+              "DEBUG_RESTART fetch of %s interrupted due to memory pressure; restarting.",
+              repositoryName);
           env.getListener()
               .post(
                   RepositoryFetchProgress.ongoing(

src/main/java/com/google/devtools/build/lib/skyframe/HighWaterMarkLimiter.java

@@ -96,6 +96,9 @@ void handle(MemoryPressureEvent event) {
           actual, threshold, remainingStat);
     }
 
+    // DEBUG ONLY: unlike the rate-limited log statements above, this records every single drop.
+    logger.atInfo().log("DEBUG_DROP dropping temporary state (fullGc=%s)", event.wasFullGc());
+
     // These caches trade temporary memory for CPU savings. Therefore if we're under memory
     // pressure, clearing them is definitely a good idea.
     skyframeExecutor.dropUnnecessaryTemporarySkyframeState();