Sp 1132: Replace FileSet adapters with FileStream adapters in all e2e tests

streampipes-extensions/streampipes-connect-adapters-iiot/src/main/java/org/apache/streampipes/connect/iiot/protocol/stream/FileStreamProtocol.java

@@ -71,6 +71,8 @@ public class FileStreamProtocol extends Protocol {
   private static final String SPEED_UP_FACTOR = "speedUpFactor";
   private static final String FASTEST = "fastest";
 
+  private static final String SPEED_UP_FACTOR_GROUP = "speed-up-factor-group";
+
 
   private static final Logger logger = LoggerFactory.getLogger(FileStreamProtocol.class);
 
@@ -300,7 +302,7 @@ public ProtocolDescription declareModel() {
             Alternatives.from(Labels.withId(KEEP_ORIGINAL_TIME), true),
             Alternatives.from(Labels.withId(FASTEST)),
             Alternatives.from(Labels.withId(SPEED_UP_FACTOR),
-                StaticProperties.group(Labels.withId("speed-up-factor-group"),
+                StaticProperties.group(Labels.withId(SPEED_UP_FACTOR_GROUP),
                     StaticProperties.doubleFreeTextProperty(Labels.withId(SPEED_UP)))))
         .build();
   }

ui/cypress/fixtures/connect/aggregationRules/expected.csv

@@ -1,2 +0,0 @@
-timestamp;value
-1623871499000;2.0

ui/cypress/support/builder/GenericAdapterBuilder.ts

@@ -76,7 +76,7 @@ export class GenericAdapterBuilder {
         return this;
     }
 
-    public setFormat(format: 'csv' | 'json_array') {
+    public setFormat(format: 'csv' | 'json_array' | 'json_object') {
         this.genericAdapterInput.format = format;
         return this;
     }

ui/cypress/support/utils/DataDownloadDialogUtils.ts

@@ -50,12 +50,10 @@ export class DataDownloadDialogUtils {
         cy.dataCy('download-configuration-next-btn').click();
 
         // Format
-        // cy.dataCy('download-configuration-json').click();
         cy.dataCy(
             `download-configuration-${exportConfig.formatExportConfig.exportFormat}`,
         ).click();
         if ('delimiter' in exportConfig.formatExportConfig) {
-            // if ((exportConfig.formatExportConfig as CsvFormatExportConfig).delimiter !== undefined) {
             cy.dataCy(
                 `download-configuration-delimiter-${
                     (exportConfig.formatExportConfig as CsvFormatExportConfig)

ui/cypress/support/utils/PrepareTestDataUtils.ts

@@ -37,13 +37,8 @@ export class PrepareTestDataUtils {
             format,
             storeInDataLake,
         );
-        // ConnectUtils.addGenericStreamAdapter(adapter, 'sp-connect-adapter-gathering-live-preview');
-        ConnectUtils.addGenericStreamAdapter(adapter);
 
-        // Wait till data is stored
-        // if (storeInDataLake) {
-        //     cy.wait(10000);
-        // }
+        ConnectUtils.addGenericStreamAdapter(adapter);
     }
 
     private static getDataLakeTestAdapter(
@@ -54,7 +49,11 @@ export class PrepareTestDataUtils {
         const adapterBuilder = GenericAdapterBuilder.create('File_Stream')
             .setName(name)
             .setTimestampProperty('timestamp')
-            .addProtocolInput('input', 'speed', '1');
+            .addProtocolInput(
+                'radio',
+                'speed',
+                'fastest_\\(ignore_original_time\\)',
+            );
 
         if (format === 'csv') {
             adapterBuilder
@@ -69,7 +68,6 @@ export class PrepareTestDataUtils {
 
         if (storeInDataLake) {
             adapterBuilder.setStoreInDataLake();
-            adapterBuilder.setStartAdapter(false);
         }
 
         return adapterBuilder.build();

ui/cypress/support/utils/ProcessingElementTestUtils.ts

@@ -24,6 +24,7 @@ import { GenericAdapterBuilder } from '../builder/GenericAdapterBuilder';
 import { PipelineBuilder } from '../builder/PipelineBuilder';
 import { PipelineElementBuilder } from '../builder/PipelineElementBuilder';
 import { ProcessorTest } from '../model/ProcessorTest';
+import { ConnectBtns } from './connect/ConnectBtns';
 
 export class ProcessingElementTestUtils {
     public static testElement(pipelineElementTest: ProcessorTest) {
@@ -47,10 +48,16 @@ export class ProcessingElementTestUtils {
         const adapterName = pipelineElementTest.name.toLowerCase();
 
         // Build adapter
-        const adapterInputBuilder = GenericAdapterBuilder.create('File_Set')
+        const adapterInputBuilder = GenericAdapterBuilder.create('File_Stream')
             .setName(adapterName)
             .setTimestampProperty('timestamp')
-            .setFormat(formatType);
+            .setFormat(formatType)
+            .setStartAdapter(false)
+            .addProtocolInput(
+                'radio',
+                'speed',
+                'fastest_\\(ignore_original_time\\)',
+            );
 
         if (formatType === 'csv') {
             adapterInputBuilder
@@ -60,7 +67,7 @@ export class ProcessingElementTestUtils {
 
         const adapterInput = adapterInputBuilder.build();
 
-        ConnectUtils.addGenericSetAdapter(adapterInput);
+        ConnectUtils.addGenericStreamAdapter(adapterInput);
 
         // Build Pipeline
         const pipelineInput = PipelineBuilder.create(pipelineElementTest.name)
@@ -76,8 +83,10 @@ export class ProcessingElementTestUtils {
 
         PipelineUtils.addPipeline(pipelineInput);
 
-        // Wait till data is stored
-        cy.wait(10000);
+        ConnectUtils.goToConnect();
+        ConnectBtns.startAdapter().click();
+
+        cy.wait(3000);
 
         DataLakeUtils.checkResults(
             dataLakeIndex,

ui/cypress/support/utils/connect/ConnectUtils.ts

@@ -30,7 +30,6 @@ import { ConnectBtns } from './ConnectBtns';
 export class ConnectUtils {
     public static testSpecificStreamAdapter(
         adapterConfiguration: SpecificAdapterInput,
-        successElement = 'sp-connect-adapter-live-preview',
     ) {
         ConnectUtils.goToConnect();
 
@@ -54,7 +53,7 @@ export class ConnectUtils {
 
         ConnectEventSchemaUtils.finishEventSchemaConfiguration();
 
-        ConnectUtils.startStreamAdapter(adapterConfiguration, successElement);
+        ConnectUtils.startStreamAdapter(adapterConfiguration);
     }
 
     public static testGenericStreamAdapter(
@@ -65,21 +64,12 @@ export class ConnectUtils {
 
     public static addGenericStreamAdapter(
         adapterConfiguration: GenericAdapterInput,
-        successElement = 'sp-connect-adapter-live-preview',
     ) {
         ConnectUtils.addGenericAdapter(adapterConfiguration);
 
-        ConnectUtils.startStreamAdapter(adapterConfiguration, successElement);
+        ConnectUtils.startStreamAdapter(adapterConfiguration);
     }
 
-    // public static addGenericSetAdapter(
-    //   adapterConfiguration: GenericAdapterInput,
-    // ) {
-    //   ConnectUtils.addGenericAdapter(adapterConfiguration);
-    //
-    //   ConnectUtils.startSetAdapter(adapterConfiguration);
-    // }
-
     public static addGenericAdapter(adapterConfiguration: GenericAdapterInput) {
         ConnectUtils.goToConnect();
 
@@ -188,23 +178,13 @@ export class ConnectUtils {
         cy.get('#event-schema-next-button').click();
     }
 
-    public static startStreamAdapter(
-        adapterInput: AdapterInput,
-        successElement = 'sp-connect-adapter-live-preview',
-    ) {
-        ConnectUtils.startAdapter(adapterInput, successElement);
+    public static startStreamAdapter(adapterInput: AdapterInput) {
+        ConnectUtils.startAdapter(adapterInput);
     }
 
-    // public static startSetAdapter(adapterInput: AdapterInput) {
-    //   ConnectUtils.startAdapter(
-    //     adapterInput,
-    //     'sp-connect-adapter-set-success',
-    //   );
-    // }
-
     public static startAdapter(
         adapterInput: AdapterInput,
-        successElement: string,
+        noLiveDataView = false,
     ) {
         // Set adapter name
         cy.dataCy('sp-adapter-name').type(adapterInput.adapterName);
@@ -225,7 +205,15 @@ export class ConnectUtils {
 
         ConnectBtns.adapterSettingsStartAdapter().click();
 
-        cy.dataCy(successElement, { timeout: 60000 }).should('be.visible');
+        if (adapterInput.startAdapter && !noLiveDataView) {
+            cy.dataCy('sp-connect-adapter-success-live-preview', {
+                timeout: 60000,
+            }).should('be.visible');
+        } else {
+            cy.dataCy('sp-connect-adapter-success-added', {
+                timeout: 60000,
+            }).should('be.visible');
+        }
 
         this.closeAdapterPreview();
     }
@@ -260,6 +248,7 @@ export class ConnectUtils {
                 'speed',
                 'fastest_\\(ignore_original_time\\)',
             )
+            .addProtocolInput('radio', 'replayonce', 'yes')
             .setName('Adapter to test rules')
             .setFormat('csv')
             .addFormatInput('input', 'delimiter', ';')
@@ -292,9 +281,9 @@ export class ConnectUtils {
         cy.get('div').contains('Values').parent().click();
 
         // Validate resulting event
-        cy.dataCy('sp-connect-adapter-live-preview', { timeout: 10000 }).should(
-            'be.visible',
-        );
+        cy.dataCy('sp-connect-adapter-success-live-preview', {
+            timeout: 10000,
+        }).should('be.visible');
 
         // validate that three event properties
         cy.get('.preview-row', { timeout: 10000 })
@@ -308,10 +297,7 @@ export class ConnectUtils {
         ignoreTime: boolean,
         waitTime = 0,
     ) {
-        ConnectUtils.startAdapter(
-            adapterConfiguration,
-            'sp-connect-adapter-gathering-live-preview',
-        );
+        ConnectUtils.startAdapter(adapterConfiguration, true);
 
         // Wait till data is stored
         cy.wait(waitTime);

ui/cypress/support/utils/datalake/DataLakeUtils.ts

@@ -34,10 +34,16 @@ export class DataLakeUtils {
         storeInDataLake: boolean = true,
         format: 'csv' | 'json_array',
     ) {
-        const adapterBuilder = GenericAdapterBuilder.create('File_Set')
+        const adapterBuilder = GenericAdapterBuilder.create('File_Stream')
             .setName(name)
             .setTimestampProperty('timestamp')
-            .addDimensionProperty('randomtext');
+            .addDimensionProperty('randomtext')
+            .addProtocolInput(
+                'radio',
+                'speed',
+                'fastest_\\(ignore_original_time\\)',
+            )
+            .setStartAdapter(true);
 
         if (format === 'csv') {
             adapterBuilder
@@ -67,12 +73,8 @@ export class DataLakeUtils {
             true,
             format,
         );
-        ConnectUtils.addGenericAdapter(adapter);
 
-        // Wait till data is stored
-        if (wait) {
-            cy.wait(10000);
-        }
+        ConnectUtils.addGenericStreamAdapter(adapter);
     }
 
     public static addDataViewAndWidget(

ui/cypress/tests/adapter/createAdapterWithoutStarting.spec.ts

@@ -33,10 +33,7 @@ describe('Creates a new adapter without starting it', () => {
             .setStartAdapter(false)
             .build();
 
-        ConnectUtils.testSpecificStreamAdapter(
-            adapterInput,
-            'sp-connect-adapter-not-started-success',
-        );
+        ConnectUtils.testSpecificStreamAdapter(adapterInput);
 
         ConnectUtils.startAndValidateAdapter(7);
 

ui/cypress/tests/adapter/formats/format.spec.ts

@@ -188,9 +188,7 @@ const navigateToFormatSelection = () => {
 
     ConnectUtils.selectAdapter('File_Stream');
 
-    ConnectUtils.configureAdapter(
-        UserInputBuilder.create().add('input', 'speed', '1').build(),
-    );
+    ConnectUtils.configureAdapter([]);
 };
 
 const validateResult = expected => {

ui/cypress/tests/adapter/rules/streamRules.spec.ts

@@ -31,14 +31,14 @@ describe('Connect aggregation rule transformations', () => {
 
         ConnectEventSchemaUtils.markPropertyAsTimestamp('timestamp');
         ConnectEventSchemaUtils.finishEventSchemaConfiguration();
+
         cy.dataCy('connect-reduce-event-rate-box').children().click();
         cy.dataCy('connect-reduce-event-input').type('2000');
 
         ConnectUtils.tearDownPreprocessingRuleTest(
             adapterConfiguration,
             'cypress/fixtures/connect/aggregationRules/expected.csv',
             false,
-            2000,
         );
     });
 });

ui/cypress/tests/dataDownloadDialog/dataDownloadDialogTest.smoke.spec.ts

@@ -28,6 +28,7 @@ describe('Test live data download dialog', () => {
             'dataDownloadDialog/input.json',
             'json_array',
         );
+
         DataLakeUtils.addDataViewAndTableWidget(dataViewName, 'Persist');
         DataLakeUtils.saveDataExplorerWidgetConfiguration();
     });

ui/cypress/tests/datalake/deleteWidget.ts

@@ -21,7 +21,6 @@ describe('Test Table View in Data Explorer', () => {
     beforeEach('Setup Test', () => {
         cy.initStreamPipesTest();
         DataLakeUtils.loadDataIntoDataLake('datalake/sample.csv', false);
-        // cy.login();
     });
 
     it('Perform Test', () => {

ui/cypress/tests/datalake/widgetDataConfiguration.smoke.spec.ts

@@ -24,8 +24,6 @@ describe('Test Table View in Data Explorer', () => {
     beforeEach('Setup Test', () => {
         cy.initStreamPipesTest();
         DataLakeUtils.loadDataIntoDataLake('datalake/sample.csv');
-        // cy.login();
-        // DataLakeUtils.goToDatalake();
     });
 
     it('Perform Test', () => {

ui/src/app/connect/components/runtime-info/pipeline-element-runtime-info.component.html

@@ -35,7 +35,7 @@ <h3>Gathering data for live preview...</h3>
         </div>
     </div>
 </div>
-<div *ngIf="runtimeData" data-cy="sp-connect-adapter-live-preview">
+<div *ngIf="runtimeData" data-cy="sp-connect-adapter-success-live-preview">
     <p>Here is a preview of your data:</p>
     <table class="dataTable row-border hover">
         <thead>

ui/src/app/connect/dialog/adapter-started/adapter-started-dialog.component.html

@@ -76,8 +76,7 @@ <h3>Adapter successfully updated</h3>
                             fxLayout="row"
                         >
                             <i class="material-icons">done</i>
-                            <span
-                                data-cy="sp-connect-adapter-not-started-success"
+                            <span data-cy="sp-connect-adapter-success-added"
                                 >&nbsp;Your new data stream is now available in
                                 the pipeline editor.</span
                             >