Test flaky

Author: Oleg Plakida

.config/nextest.toml

@@ -16,7 +16,7 @@ dir = "target/nextest"
 # * retries = 3
 # * retries = { backoff = "fixed", count = 2, delay = "1s" }
 # * retries = { backoff = "exponential", count = 10, delay = "1s", jitter = true, max-delay = "10s" }
-retries = 0
+retries = 5
 
 # The number of threads to run tests with. Supported values are either an integer or
 # the string "num-cpus". Can be overridden through the `--test-threads` option.
@@ -94,7 +94,7 @@ path = "junit.xml"
 # The name of the top-level "report" element in JUnit report. If aggregating
 # reports across different test runs, it may be useful to provide separate names
 # for each report.
-report-name = "nextest-run"
+report-name = "substrate"
 
 # Whether standard output and standard error for passing tests should be stored in the JUnit report.
 # Output is stored in the <system-out> and <system-err> elements of the <testcase> element.

Cargo.lock

@@ -26,6 +26,7 @@ sp-std = { version = "5.0.0", default-features = false, path = "../../../primiti
 sp-version = { version = "5.0.0", default-features = false, path = "../../../primitives/version" }
 trybuild = { version = "1.0.74", features = [ "diff" ] }
 pretty_assertions = "1.2.1"
+rand = "0.8.5"
 rustversion = "1.0.6"
 frame-system = { version = "4.0.0-dev", default-features = false, path = "../../system" }
 # The "std" feature for this pallet is never activated on purpose, in order to test construct_runtime error message

frame/support/test/Cargo.toml

@@ -15,22 +15,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use rand::Rng;
+
 #[rustversion::attr(not(stable), ignore)]
 #[cfg(not(feature = "disable-ui-tests"))]
 #[test]
 fn benchmark_ui() {
-	// Only run the ui tests when `RUN_UI_TESTS` is set.
-	if std::env::var("RUN_UI_TESTS").is_err() {
-		return
-	}
-
-	// As trybuild is using `cargo check`, we don't need the real WASM binaries.
-	std::env::set_var("SKIP_WASM_BUILD", "1");
-
-	// Deny all warnings since we emit warnings as part of a Pallet's UI.
-	std::env::set_var("RUSTFLAGS", "--deny warnings");
-
-	let t = trybuild::TestCases::new();
-	t.compile_fail("tests/benchmark_ui/*.rs");
-	t.pass("tests/benchmark_ui/pass/*.rs");
+	let mut rng = rand::thread_rng();
+	assert_eq!(rng.gen_range(0..6), 0);
 }

frame/support/test/tests/benchmark_ui.rs

@@ -234,6 +234,22 @@ test-linux-stable:
     reports:
       junit: target/nextest/default/junit.xml
 
+test-linux-stable-upload-test-results:
+  stage: test
+  needs:
+    - job: test-linux-stable
+      artifacts: true
+  extends:
+    - .docker-env
+    - .test-refs
+  script:
+    - cat target/nextest/default/junit.xml | xq . > target/nextest/default/junit.json
+    - curl -v -XPOST --http1.1
+      -u ${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}
+      https://elasticsearch.parity-build.parity.io/unit-tests/_doc/
+      -H 'Content-Type: application/json'
+      -d @target/nextest/default/junit.json
+
 test-frame-support:
   stage: test
   extends: