disable examples check (apache#27126)

* disable examples check

* Clean up (apache#27125)

* Fix a Gradle task (apache#27125)

* Fix a SCIO example descriptor (apache#27125)

* Test local code instead of fetching from GitHub (apache#27125)

* Fix tests (apache#27125)

* Fix tests (apache#27125)

* Update README (apache#27125)

* Re-run tests (apache#27125)

---------

Co-authored-by: Alexey Inkin <alexey.inkin@akvelon.com>

Author: 2 people

playground/frontend/README.md

@@ -178,6 +178,18 @@ flutter test
 ./gradlew :playground:frontend:integrationTest -PdeviceId=web-server
 ```
 
+By default, tests do not expect specific code and output from most examples.
+This is because we get the expected example files from GitHub itself at runtime.
+Examples in the default GitHub branch may differ from the deployed ones,
+and this will break the tests.
+
+To expect specific code and output, run tests like this using any repository owner/name
+and commit reference to load the examples from:
+
+```bash
+./gradlew :playground:frontend:integrationTest -PexamplesRepository=apache/beam -PexamplesRef=master
+```
+
 ## Localization
 
 The project is in the process of migrating from

playground/frontend/build.gradle

@@ -263,20 +263,33 @@ tasks.register("integrationTest_standalone_share_code") {
 
 void runIntegrationTest(String path, String url) {
   // Run with -PdeviceId=web-server for headless tests.
+  // Run with -PexamplesRepository=apache/beam -PexamplesRef=master to verify that deployed examples match the given branch or commit.
   String deviceId = project.hasProperty("deviceId") ? project.deviceId : "chrome"
+  String examplesRepository = project.hasProperty("examplesRepository") ? project.examplesRepository : null
+  String examplesRef = project.hasProperty("examplesRef") ? project.examplesRef : null
+
+  List<String> flutterArgs = [
+    "drive",
+    "--driver=test_driver/integration_test.dart",
+    "--target=integration_test/${path}_test.dart",
+    "--web-launch-url='$url'",
+    "--device-id=$deviceId",
+  ]
+
+  if (examplesRepository != null) {
+    flutterArgs.add("--dart-define=examples-repository=$examplesRepository")
+  }
+  if (examplesRef != null) {
+    flutterArgs.add("--dart-define=examples-ref=$examplesRef")
+  }
 
   exec {
     executable("flutter")
-    args(
-      "drive",
-      "--driver=test_driver/integration_test.dart",
-      "--target=integration_test/${path}_test.dart",
-      "--web-launch-url='$url'",
-      "--device-id=$deviceId",
-    )
+    args(flutterArgs)
   }
 }
 
+
 task copyDockerfileDependencies(type: Copy) {
    group = "build"
    description = "Copy files that required to build docker container"

playground/frontend/integration_test/initial_urls_test.dart

@@ -49,7 +49,7 @@ void main() {
     await _testCatalogDefaultExampleLoader(wt);
     await _testContentExampleLoader(wt);
     await _testEmptyExampleLoader(wt);
-    await _testHttpExampleLoader(wt);
+    await _testHttpExampleLoaderIfDeployed(wt);
     await _testStandardExampleLoader(wt);
     await _testUserSharedExampleLoader(wt);
 
@@ -76,7 +76,7 @@ final _croppedViewOptions = _mapToQueryString(_croppedViewOptionsMap);
 Future<void> _testEmbeddedRoot(WidgetTester wt) async {
   await wt.navigateAndSettle(_embeddedPath);
   expectSdk(Sdk.java, wt);
-  expectVisibleText('', wt);
+  expectVisibleTextIfDeployed('', wt);
   expectLastAnalyticsEvent(
     const LoadedAnalyticsEvent(
       sdk: Sdk.java,
@@ -90,7 +90,7 @@ Future<void> _testStandaloneRoot(WidgetTester wt) async {
   await wt.navigateAndSettle(_standalonePath);
 
   expectSdk(Sdk.java, wt);
-  expectVisibleText(visibleText, wt);
+  expectVisibleTextIfDeployed(visibleText, wt);
   expectLastAnalyticsEvent(
     const LoadedAnalyticsEvent(
       sdk: Sdk.java,
@@ -102,7 +102,7 @@ Future<void> _testStandaloneRoot(WidgetTester wt) async {
 Future<void> _testEmbeddedSdkOnly(WidgetTester wt) async {
   await wt.navigateAndSettle('$_embeddedPath?sdk=go');
   expectSdk(Sdk.go, wt);
-  expectVisibleText('', wt);
+  expectVisibleTextIfDeployed('', wt);
   expectLastAnalyticsEvent(
     const LoadedAnalyticsEvent(
       sdk: Sdk.go,
@@ -116,7 +116,7 @@ Future<void> _testStandaloneSdkOnly(WidgetTester wt) async {
   await wt.navigateAndSettle('$_standalonePath?sdk=go');
 
   expectSdk(Sdk.go, wt);
-  expectVisibleText(visibleText, wt);
+  expectVisibleTextIfDeployed(visibleText, wt);
   expectLastAnalyticsEvent(
     const LoadedAnalyticsEvent(
       sdk: Sdk.go,
@@ -130,7 +130,7 @@ Future<void> _testCatalogDefaultExampleLoader(WidgetTester wt) async {
   await wt.navigateAndSettle('$_standalonePath?sdk=go&default=true');
 
   expectSdk(Sdk.go, wt);
-  expectVisibleText(visibleText, wt);
+  expectVisibleTextIfDeployed(visibleText, wt);
   expectLastAnalyticsEvent(
     const LoadedAnalyticsEvent(
       sdk: Sdk.go,
@@ -150,7 +150,7 @@ Future<void> _testContentExampleLoader(WidgetTester wt) async {
       '$path?sdk=go&files=${Uri.encodeComponent(files)}&$_fullViewOptions',
     );
     expectSdk(Sdk.go, wt);
-    expectVisibleText(goExample.foldedVisibleText, wt);
+    expectVisibleTextIfDeployed(goExample.foldedVisibleText, wt);
     expectLastAnalyticsEvent(
       const LoadedAnalyticsEvent(
         sdk: Sdk.go,
@@ -163,7 +163,7 @@ Future<void> _testContentExampleLoader(WidgetTester wt) async {
       '$path?sdk=go&files=${Uri.encodeComponent(files)}&$_croppedViewOptions',
     );
     expectSdk(Sdk.go, wt);
-    expectVisibleText(goExample.croppedFoldedVisibleText, wt);
+    expectVisibleTextIfDeployed(goExample.croppedFoldedVisibleText, wt);
     _expectReadOnly(wt);
   }
 }
@@ -172,7 +172,7 @@ Future<void> _testEmptyExampleLoader(WidgetTester wt) async {
   for (final path in _paths) {
     await wt.navigateAndSettle('$path?sdk=go&empty=true');
     expectSdk(Sdk.go, wt);
-    expectVisibleText('', wt);
+    expectVisibleTextIfDeployed('', wt);
     expectLastAnalyticsEvent(
       const LoadedAnalyticsEvent(
         sdk: Sdk.go,
@@ -182,13 +182,17 @@ Future<void> _testEmptyExampleLoader(WidgetTester wt) async {
   }
 }
 
-Future<void> _testHttpExampleLoader(WidgetTester wt) async {
+Future<void> _testHttpExampleLoaderIfDeployed(WidgetTester wt) async {
+  if (!areExamplesDeployed) {
+    return;
+  }
+
   for (final path in _paths) {
     await wt.navigateAndSettle(
       '$path?sdk=go&url=${goExample.rawUrl}&$_fullViewOptions',
     );
     expectSdk(Sdk.go, wt);
-    expectVisibleText(goExample.foldedVisibleText, wt);
+    expectVisibleTextIfDeployed(goExample.foldedVisibleText, wt);
     expectLastAnalyticsEvent(
       LoadedAnalyticsEvent(
         sdk: Sdk.go,
@@ -201,7 +205,7 @@ Future<void> _testHttpExampleLoader(WidgetTester wt) async {
       '$path?sdk=go&url=${goExample.rawUrl}&$_croppedViewOptions',
     );
     expectSdk(Sdk.go, wt);
-    expectVisibleText(goExample.croppedFoldedVisibleText, wt);
+    expectVisibleTextIfDeployed(goExample.croppedFoldedVisibleText, wt);
     expectLastAnalyticsEvent(
       LoadedAnalyticsEvent(
         sdk: Sdk.go,
@@ -220,7 +224,7 @@ Future<void> _testStandardExampleLoader(WidgetTester wt) async {
       '$path?sdk=go&path=${goWordCount.dbPath}',
     );
     expectSdk(Sdk.go, wt);
-    expectVisibleText(visibleText, wt);
+    expectVisibleTextIfDeployed(visibleText, wt);
     expectLastAnalyticsEvent(
       LoadedAnalyticsEvent(
         sdk: Sdk.go,
@@ -234,22 +238,14 @@ Future<void> _testUserSharedExampleLoader(WidgetTester wt) async {
   final template = await goExample.getFullText();
   final tail = '\n//${DateTime.now().millisecondsSinceEpoch}';
   final content = '$template$tail';
-
-  final exampleCache = wt.findPlaygroundController().exampleCache;
-  final snippetId = await exampleCache.saveSnippet(
-    files: [SnippetFile(content: content, isMain: false, name: 'name')],
-    sdk: Sdk.go,
-    pipelineOptions: '--name=value',
-  );
-
-  print('Created user-shared example ID: $snippetId');
+  final snippetId = await _getSnippetId(wt, content, Sdk.go);
 
   for (final path in _paths) {
     await wt.navigateAndSettle(
       '$path?sdk=go&shared=$snippetId&$_fullViewOptions',
     );
     expectSdk(Sdk.go, wt);
-    expectVisibleText('${goExample.foldedVisibleText}$tail', wt);
+    expectVisibleTextIfDeployed('${goExample.foldedVisibleText}$tail', wt);
     expectLastAnalyticsEvent(
       LoadedAnalyticsEvent(
         sdk: Sdk.go,
@@ -262,7 +258,7 @@ Future<void> _testUserSharedExampleLoader(WidgetTester wt) async {
       '$path?sdk=go&shared=$snippetId&$_croppedViewOptions',
     );
     expectSdk(Sdk.go, wt);
-    expectVisibleText(goExample.croppedFoldedVisibleText, wt);
+    expectVisibleTextIfDeployed(goExample.croppedFoldedVisibleText, wt);
     expectLastAnalyticsEvent(
       LoadedAnalyticsEvent(
         sdk: Sdk.go,
@@ -273,18 +269,38 @@ Future<void> _testUserSharedExampleLoader(WidgetTester wt) async {
   }
 }
 
+Future<String> _getSnippetId(WidgetTester wt, String content, Sdk sdk) async {
+  final exampleCache = wt.findPlaygroundController().exampleCache;
+  final snippetId = await exampleCache.saveSnippet(
+    files: [SnippetFile(content: content, isMain: false, name: 'name')],
+    sdk: sdk,
+    pipelineOptions: '--name=value',
+  );
+
+  print('Created user-shared example ID: $snippetId');
+
+  return snippetId;
+}
+
 Future<void> _testMultipleExamples(WidgetTester wt) async {
-  final javaVisibleText = await javaAggregationMax.getVisibleText();
-  final goVisibleText = goExample.foldedVisibleText;
+  final javaTemplate = await javaExample.getFullText();
+  final goTemplate= await goExample.getFullText();
+  final tail = '\n//${DateTime.now().millisecondsSinceEpoch}';
+  final javaSnippetId = await _getSnippetId(wt, '$javaTemplate$tail', Sdk.java);
+  final goSnippetId = await _getSnippetId(wt, '$goTemplate$tail', Sdk.go);
+
+  final javaVisibleText = '${javaExample.foldedVisibleText}$tail';
+  final goVisibleText = '${goExample.foldedVisibleText}$tail';
 
   final examplesList = [
     {
       'sdk': Sdk.java.id,
-      'path': javaAggregationMax.dbPath,
+      'shared': javaSnippetId,
+      ..._fullViewOptionsMap,
     },
     {
       'sdk': Sdk.go.id,
-      'url': goExample.rawUrl,
+      'shared': goSnippetId,
       ..._fullViewOptionsMap,
     },
   ];
@@ -293,11 +309,11 @@ Future<void> _testMultipleExamples(WidgetTester wt) async {
   for (final path in _paths) {
     await wt.navigateAndSettle('$path?sdk=go&examples=$examples');
     expectSdk(Sdk.go, wt);
-    expectVisibleText(goVisibleText, wt);
+    expectVisibleText(goVisibleText, wt, reason: 'go, $path');
     expectLastAnalyticsEvent(
       LoadedAnalyticsEvent(
         sdk: Sdk.go,
-        snippet: goExample.rawUrl,
+        snippet: goSnippetId,
       ),
     );
     await _expectEditableAndReadOnly(wt);
@@ -307,7 +323,8 @@ Future<void> _testMultipleExamples(WidgetTester wt) async {
     await wt.pumpAndSettle();
 
     expectSdk(Sdk.java, wt);
-    expectVisibleText(javaVisibleText, wt);
+    expectVisibleText(javaVisibleText, wt, reason: 'java, $path');
+    await _expectEditableAndReadOnly(wt);
   }
 }
 

playground/frontend/integration_test/miscellaneous_ui/external_url_navigation.dart

@@ -90,6 +90,9 @@ Future<void> _checkMenuItems(WidgetTester wt) async {
 }
 
 Future<void> _checkExampleDescription(WidgetTester wt) async {
+  final url = wt.findPlaygroundController().selectedExample?.urlVcs;
+  expect(url, isNotNull);
+
   await _tapAndExpectNavigationEvent(
     wt,
     [
@@ -99,7 +102,7 @@ Future<void> _checkExampleDescription(WidgetTester wt) async {
             matching: find.byType(GithubButton),
           ),
     ],
-    javaMinimalWordCount.url,
+    url!,
   );
 }
 

playground/frontend/integration_test/standalone_change_example_sdk_run_test.dart

@@ -35,12 +35,17 @@ void main() {
     await wt.tapAndSettle(find.exampleSelector());
     await wt.tapAndSettle(find.exampleItemInDropdown(javaAggregationMax.name));
 
-    expect(
-      wt.findOneCodeController().lastTextSpan!.toPlainText().isAsIfCutFrom(
-            await javaAggregationMax.getVisibleText(),
-          ),
-      true,
-    );
+    if (areExamplesDeployed) {
+      final visibleText = await javaAggregationMax.getVisibleText();
+      final lastSpanText =
+          wt.findOneCodeController().lastTextSpan!.toPlainText();
+
+      expect(
+        lastSpanText.isAsIfCutFrom(visibleText),
+        true,
+        reason: '$lastSpanText is not as if cut from $visibleText',
+      );
+    }
 
     expectLastAnalyticsEvent(
       SnippetSelectedAnalyticsEvent(
@@ -65,18 +70,25 @@ public class MyClass {
 
     await wt.tapAndSettle(find.runOrCancelButton());
 
-    expectOutputEquals('$_outputPrefix$text', wt);
+    expectOutputEqualsIfDeployed('$_outputPrefix$text', wt);
   }
 
   Future<void> switchToPython(WidgetTester wt) async {
     await wt.changeSdk(Sdk.python);
 
-    expect(
-      wt.findOneCodeController().lastTextSpan!.toPlainText().isAsIfCutFrom(
-            await pythonWordCountWithMetrics.getVisibleText(),
-          ),
-      true,
-    );
+    if (areExamplesDeployed) {
+      final visibleText = await pythonWordCountWithMetrics.getVisibleText();
+      final lastSpanText = wt
+          .findOneCodeController()
+          .lastTextSpan!
+          .toPlainText();
+
+      expect(
+        lastSpanText.isAsIfCutFrom(visibleText),
+        true,
+        reason: '$lastSpanText is not as if cut from $visibleText',
+      );
+    }
 
     expectLastAnalyticsEvent(const SdkSelectedAnalyticsEvent(sdk: Sdk.python));
   }
@@ -108,7 +120,7 @@ public class MyClass {
 
     await wt.tapAndSettle(find.runOrCancelButton());
 
-    expectOutputEquals('$_outputPrefix$text', wt);
+    expectOutputEqualsIfDeployed('$_outputPrefix$text', wt);
   }
 
   testWidgets('Change example, change SDK, run', (WidgetTester wt) async {

playground/frontend/integration_test/standalone_default_examples_test.dart

@@ -53,7 +53,7 @@ Future<void> _expectExample(ExampleDescriptor example, WidgetTester wt) async {
   final visibleText = await example.getVisibleText();
 
   expectSdk(example.sdk, wt);
-  expectVisibleText(visibleText, wt);
+  expectVisibleTextIfDeployed(visibleText, wt);
 
   if (example.hasGraphTab) {
     expect(find.graphTab(), findsOneWidget);