Merge pull request #405 from magento/develop

MFTF 2.4.3 - Merge to Master

Author: okolesnyk

.gitignore

@@ -16,4 +16,6 @@ dev/tests/functional/MFTF.suite.yml
 dev/tests/functional/_output
 dev/mftf.log
 dev/tests/mftf.log
-dev/tests/docs/*
+dev/tests/docs/*
+dev/tests/_output
+dev/tests/functional.suite.yml

CHANGELOG.md

@@ -1,6 +1,22 @@
 Magento Functional Testing Framework Changelog
 ================================================
 
+2.4.3
+-----
+* Customizability
+    * Use of `_CREDS` has been extended to `<magentoCLI>` and `<createData>` actions
+* Traceability
+    * A Test step is now generated and injected in the allure report when a test is reported as `BROKEN`. 
+
+### Fixes
+* `static-checks` command now properly returns `1` if any static check failed.
+* MFTF Console Printer class correctly utilizes `--steps` and other flags passed directly to `codecept commands`.
+* `*source` actions correctly print when using `userInput` or `html` attributes.
+* XML Comments should no longer throw an error in parsing when used outside `test/actionGroup`
+
+### GitHub Issues/Pull requests:
+* [#703](https://github.com/magento/magento2-functional-testing-framework/pull/403) -- SMALL_CHANGE: Minor change suggested
+
 2.4.2
 -----
 * Traceability

bin/mftf

@@ -29,7 +29,7 @@ try {
 try {
     $application = new Symfony\Component\Console\Application();
     $application->setName('Magento Functional Testing Framework CLI');
-    $application->setVersion('2.4.2');
+    $application->setVersion('2.4.3');
     /** @var \Magento\FunctionalTestingFramework\Console\CommandListInterface $commandList */
     $commandList = new \Magento\FunctionalTestingFramework\Console\CommandList;
     foreach ($commandList->getCommands() as $command) {

composer.json

@@ -2,7 +2,7 @@
     "name": "magento/magento2-functional-testing-framework",
     "description": "Magento2 Functional Testing Framework",
     "type": "library",
-    "version": "2.4.2",
+    "version": "2.4.3",
     "license": "AGPL-3.0",
     "keywords": ["magento", "automation", "functional", "testing"],
     "config": {

composer.lock

@@ -8,6 +8,8 @@
 use AspectMock\Test as AspectMock;
 use Magento\FunctionalTestingFramework\DataGenerator\Handlers\DataObjectHandler;
 use Magento\FunctionalTestingFramework\DataGenerator\Objects\EntityDataObject;
+use Magento\FunctionalTestingFramework\Exceptions\TestReferenceException;
+use Magento\FunctionalTestingFramework\Exceptions\XmlException;
 use Magento\FunctionalTestingFramework\Test\Objects\ActionObject;
 use Magento\FunctionalTestingFramework\Test\Util\ActionMergeUtil;
 use Magento\FunctionalTestingFramework\Test\Util\ActionObjectExtractor;
@@ -100,7 +102,7 @@ public function testResolveActionStepEntityData()
         $dataFieldName = 'myfield';
         $dataFieldValue = 'myValue';
         $userInputKey = "userInput";
-        $userinputValue = "{{" . "${dataObjectName}.${dataFieldName}}}";
+        $userInputValue = "{{" . "${dataObjectName}.${dataFieldName}}}";
         $actionName = "myAction";
         $actionType = "myCustomType";
 
@@ -113,10 +115,10 @@ public function testResolveActionStepEntityData()
         AspectMock::double(DataObjectHandler::class, ['getInstance' => $mockDOHInstance]);
 
         // Create test object and action object
-        $actionAttributes = [$userInputKey => $userinputValue];
+        $actionAttributes = [$userInputKey => $userInputValue];
         $actions[$actionName] = new ActionObject($actionName, $actionType, $actionAttributes);
 
-        $this->assertEquals($userinputValue, $actions[$actionName]->getCustomActionAttributes()[$userInputKey]);
+        $this->assertEquals($userInputValue, $actions[$actionName]->getCustomActionAttributes()[$userInputKey]);
 
         $mergeUtil = new ActionMergeUtil("test", "TestCase");
         $resolvedActions = $mergeUtil->resolveActionSteps($actions);
@@ -127,8 +129,14 @@ public function testResolveActionStepEntityData()
     /**
      * Verify that an XmlException is thrown when an action references a non-existant action.
      *
+     * @throws TestReferenceException
+     * @throws XmlException
      * @return void
      */
+    /**
+     * @throws TestReferenceException
+     * @throws XmlException
+     */
     public function testNoActionException()
     {
         $actionObjects = [];
@@ -151,6 +159,8 @@ public function testNoActionException()
     /**
      * Verify that a <waitForPageLoad> action is added after actions that have a wait (timeout property).
      *
+     * @throws TestReferenceException
+     * @throws XmlException
      * @return void
      */
     public function testInsertWait()
@@ -173,6 +183,111 @@ public function testInsertWait()
         $this->assertEquals($expected, $actual);
     }
 
+    /**
+     * Verify that a <fillField> action is replaced by <fillSecretField> when secret _CREDS are referenced.
+     *
+     * @throws TestReferenceException
+     * @throws XmlException
+     */
+    public function testValidFillFieldSecretFunction()
+    {
+        $actionObjectOne = new ActionObject(
+            'actionKey1',
+            'fillField',
+            ['userInput' => '{{_CREDS.username}}']
+        );
+        $actionObject = [$actionObjectOne];
+
+        $actionMergeUtil = new ActionMergeUtil('actionMergeUtilTest', 'TestCase');
+
+        $result = $actionMergeUtil->resolveActionSteps($actionObject);
+
+        $expectedValue = new ActionObject(
+            'actionKey1',
+            'fillSecretField',
+            ['userInput' => '{{_CREDS.username}}']
+        );
+        $this->assertEquals($expectedValue, $result['actionKey1']);
+    }
+
+    /**
+     * Verify that a <magentoCLI> action uses <magentoCLI> when secret _CREDS are referenced.
+     *
+     * @throws TestReferenceException
+     * @throws XmlException
+     */
+    public function testValidMagentoCLISecretFunction()
+    {
+        $actionObjectOne = new ActionObject(
+            'actionKey1',
+            'magentoCLI',
+            ['command' => 'config:set cms/wysiwyg/enabled {{_CREDS.payment_authorizenet_login}}']
+        );
+        $actionObject = [$actionObjectOne];
+
+        $actionMergeUtil = new ActionMergeUtil('actionMergeUtilTest', 'TestCase');
+
+        $result = $actionMergeUtil->resolveActionSteps($actionObject);
+
+        $expectedValue = new ActionObject(
+            'actionKey1',
+            'magentoCLISecret',
+            ['command' => 'config:set cms/wysiwyg/enabled {{_CREDS.payment_authorizenet_login}}']
+        );
+        $this->assertEquals($expectedValue, $result['actionKey1']);
+    }
+
+    /**
+     * Verify that a <field> override in a <createData> action uses <field> when secret _CREDS are referenced.
+     *
+     * @throws TestReferenceException
+     * @throws XmlException
+     */
+    public function testValidCreateDataSecretFunction()
+    {
+        $actionObjectOne = new ActionObject(
+            'actionKey1',
+            'field',
+            ['value' => '{{_CREDS.payment_authorizenet_login}}']
+        );
+        $actionObject = [$actionObjectOne];
+
+        $actionMergeUtil = new ActionMergeUtil('actionMergeUtilTest', 'TestCase');
+
+        $result = $actionMergeUtil->resolveActionSteps($actionObject);
+
+        $expectedValue = new ActionObject(
+            'actionKey1',
+            'field',
+            ['value' => '{{_CREDS.payment_authorizenet_login}}']
+        );
+        $this->assertEquals($expectedValue, $result['actionKey1']);
+    }
+
+    /**
+     * Verify that a <click> action throws an exception when secret _CREDS are referenced.
+     *
+     * @throws TestReferenceException
+     * @throws XmlException
+     */
+    public function testInvalidSecretFunctions()
+    {
+        $this->expectException(TestReferenceException::class);
+        $this->expectExceptionMessage(
+            'You cannot reference secret data outside of the fillField, magentoCLI and createData actions'
+        );
+
+        $actionObjectOne = new ActionObject(
+            'actionKey1',
+            'click',
+            ['userInput' => '{{_CREDS.username}}']
+        );
+        $actionObject = [$actionObjectOne];
+
+        $actionMergeUtil = new ActionMergeUtil('actionMergeUtilTest', 'TestCase');
+        $actionMergeUtil->resolveActionSteps($actionObject);
+    }
+
     /**
      * After class functionality
      * @return void

dev/tests/unit/Magento/FunctionalTestFramework/Test/Util/ActionMergeUtilTest.php

@@ -31,7 +31,7 @@ class ActionGroupUsingCreateDataCest
 			"hook",
 			"ApiCategory",
 			[],
-			null
+			[]
 		);
 
 		$I->comment("[createConfigProductKey1] create 'ApiConfigurableProduct' entity");
@@ -40,7 +40,7 @@ class ActionGroupUsingCreateDataCest
 			"hook",
 			"ApiConfigurableProduct",
 			["createCategoryKey1"],
-			null
+			[]
 		);
 
 		$I->comment("Exiting Action Group [Key1] actionGroupWithCreateData");

dev/tests/verification/Resources/ActionGroupUsingCreateData.txt

@@ -31,7 +31,7 @@ class ActionGroupWithDataOverrideTestCest
 			"hook",
 			"ReplacementPerson",
 			[],
-			null
+			[]
 		);
 
 		$I->comment("Entering Action Group [beforeGroup] FunctionalActionGroup");

dev/tests/verification/Resources/ActionGroupWithDataOverrideTest.txt

@@ -31,7 +31,7 @@ class ActionGroupWithDataTestCest
 			"hook",
 			"ReplacementPerson",
 			[],
-			null
+			[]
 		);
 
 		$I->comment("Entering Action Group [beforeGroup] FunctionalActionGroup");

dev/tests/verification/Resources/ActionGroupWithDataTest.txt

@@ -31,7 +31,7 @@ class ActionGroupWithNoDefaultTestCest
 			"hook",
 			"ReplacementPerson",
 			[],
-			null
+			[]
 		);
 
 		$I->comment("Entering Action Group [beforeGroup] FunctionalActionGroup");

dev/tests/verification/Resources/ActionGroupWithNoDefaultTest.txt

@@ -31,7 +31,7 @@ class ActionGroupWithPersistedDataCest
 			"hook",
 			"ReplacementPerson",
 			[],
-			null
+			[]
 		);
 
 		$I->comment("Entering Action Group [beforeGroup] FunctionalActionGroup");
@@ -78,7 +78,7 @@ class ActionGroupWithPersistedDataCest
 			"test",
 			"DefaultPerson",
 			[],
-			null
+			[]
 		);
 
 		$I->comment("Entering Action Group [actionGroupWithPersistedData1] FunctionalActionGroupWithData");

dev/tests/verification/Resources/ActionGroupWithPersistedData.txt

@@ -34,7 +34,7 @@ class ActionGroupWithStepKeyReferencesCest
 			"test",
 			"simpleData",
 			[],
-			null
+			[]
 		);
 
 		$grabTextDataActionGroup = $I->grabTextFrom(".class"); // stepKey: grabTextDataActionGroup
@@ -46,7 +46,6 @@ class ActionGroupWithStepKeyReferencesCest
 		$action3ActionGroup = $I->executeJS($action3ActionGroup); // stepKey: action3ActionGroup
 		$action4ActionGroup = $I->magentoCLI($action4ActionGroup, "\"stuffHere\""); // stepKey: action4ActionGroup
 		$I->comment($action4ActionGroup);
-
 		$date = new \DateTime();
 		$date->setTimestamp(strtotime("{$action5}"));
 		$date->setTimezone(new \DateTimeZone("America/Los_Angeles"));
@@ -82,7 +81,7 @@ class ActionGroupWithStepKeyReferencesCest
 			"test",
 			"{$action10}",
 			[],
-			null
+			[]
 		);
 
 		$action11ActionGroup = $I->grabAttributeFrom($action11ActionGroup, "someInput"); // stepKey: action11ActionGroup

dev/tests/verification/Resources/ActionGroupWithStepKeyReferences.txt

@@ -31,7 +31,7 @@ class ActionGroupWithTopLevelPersistedDataCest
 			"hook",
 			"ReplacementPerson",
 			[],
-			null
+			[]
 		);
 
 		$I->comment("Entering Action Group [beforeGroup] FunctionalActionGroup");

dev/tests/verification/Resources/ActionGroupWithTopLevelPersistedData.txt

@@ -31,7 +31,7 @@ class ArgumentWithSameNameAsElementCest
 			"hook",
 			"ReplacementPerson",
 			[],
-			null
+			[]
 		);
 
 		$I->comment("Entering Action Group [beforeGroup] FunctionalActionGroup");

dev/tests/verification/Resources/ArgumentWithSameNameAsElement.txt

@@ -30,7 +30,7 @@ class AssertTestCest
 			"hook",
 			"ReplacementPerson",
 			[],
-			null
+			[]
 		);
 
 	}
@@ -50,7 +50,7 @@ class AssertTestCest
 			"test",
 			"UniquePerson",
 			[],
-			null
+			[]
 		);
 
 		$grabTextFrom1 = $I->grabTextFrom(".copyright>span"); // stepKey: grabTextFrom1

dev/tests/verification/Resources/AssertTest.txt

@@ -31,7 +31,7 @@ class BasicActionGroupTestCest
 			"hook",
 			"ReplacementPerson",
 			[],
-			null
+			[]
 		);
 
 		$I->comment("Entering Action Group [beforeGroup] FunctionalActionGroup");

dev/tests/verification/Resources/BasicActionGroupTest.txt

@@ -92,8 +92,14 @@ class BasicFunctionalTestCest
 		$I->dontSeeElementInDOM(".functionalTestSelector"); // stepKey: dontSeeElementInDOMKey1
 		$I->dontSeeInCurrentUrl("/functionalUrl"); // stepKey: dontSeeInCurrentUrlKey1
 		$I->dontSeeInField(".functionalTestSelector"); // stepKey: dontSeeInFieldKey1
-		$I->dontSeeInPageSource("someInput"); // stepKey: dontSeeInPageSourceKey1
-		$I->dontSeeInSource("<myHtmlHere>"); // stepKey: dontSeeInSourceKey1
+		$I->dontSeeInPageSource("Cosmo Kramer"); // stepKey: dontSeeInPageSourceKey1
+		$I->dontSeeInPageSource("<p>Jerry Seinfeld</p>"); // stepKey: dontSeeInPageSourceKey2
+		$I->dontSeeInPageSource("Cosmo Kramer"); // stepKey: dontSeeInPageSourceKey3
+		$I->dontSeeInPageSource("<p>Jerry Seinfeld</p>"); // stepKey: dontSeeInPageSourceKey4
+		$I->dontSeeInPageSource("foo"); // stepKey: dontSeeInPageSourceKey5
+		$I->dontSeeInPageSource("<p>foo</p>"); // stepKey: dontSeeInPageSourceKey6
+		$I->dontSeeInSource("Cosmo Kramer"); // stepKey: dontSeeInSourceKey1
+		$I->dontSeeInSource("<p>Jerry Seinfeld</p>"); // stepKey: dontSeeInSourceKey2
 		$I->dontSeeInTitle("someInput"); // stepKey: dontSeeInTitleKey1
 		$I->dontSeeLink("someInput", "/functionalUrl"); // stepKey: dontSeeLinkKey1
 		$I->dontSeeOptionIsSelected(".functionalTestSelector", "someInput"); // stepKey: dontSeeOptionIsSelectedKey1
@@ -121,7 +127,6 @@ class BasicFunctionalTestCest
 		$grabValueFromKey1 = $I->grabValueFrom(".functionalTestSelector"); // stepKey: grabValueFromKey1
 		$magentoCli1 = $I->magentoCLI("maintenance:enable", "\"stuffHere\""); // stepKey: magentoCli1
 		$I->comment($magentoCli1);
-
 		$I->makeScreenshot("screenShotInput"); // stepKey: makeScreenshotKey1
 		$I->maximizeWindow(); // stepKey: maximizeWindowKey1
 		$I->moveBack(); // stepKey: moveBackKey1
@@ -147,9 +152,11 @@ class BasicFunctionalTestCest
 		$I->seeElementInDOM(".functionalTestSelector"); // stepKey: seeElementInDOMKey1
 		$I->seeInCurrentUrl("/functionalUrl"); // stepKey: seeInCurrentUrlKey1
 		$I->seeInField(".functionalTestSelector", "someInput"); // stepKey: seeInFieldKey1
-		$I->seeInPageSource("<myHtmlHere>"); // stepKey: seeInPageSourceKey1
+		$I->seeInPageSource("Home Page"); // stepKey: seeInPageSourceKey1
+		$I->seeInPageSource("<h1 class=\"page-title\">"); // stepKey: seeInPageSourceKey2
 		$I->seeInPopup("someInput"); // stepKey: seeInPopupKey1
-		$I->seeInSource("<myHtmlHere>"); // stepKey: seeInSourceKey1
+		$I->seeInSource("Home Page"); // stepKey: seeInSourceKey1
+		$I->seeInSource("<h1 class=\"page-title\">"); // stepKey: seeInSourceKey2
 		$I->seeInTitle("someInput"); // stepKey: seeInTitleKey1
 		$I->seeLink("someInput", "/functionalUrl"); // stepKey: seeLinkKey1
 		$I->seeNumberOfElements(".functionalTestSelector"); // stepKey: seeNumberOfElementsKey1