Skip to content

Commit 651db4f

Browse files
tomreecetomreece
authored
Merge pull request #790 from magento/master
Merge master into develop
2 parents 17a8961 + 8a106ea commit 651db4f

File tree

14 files changed

+127
-41
lines changed

14 files changed

+127
-41
lines changed

‎CHANGELOG.md

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,41 @@
11
Magento Functional Testing Framework Changelog
22
================================================
3+
3.1.0
4+
________
5+
6+
### Enhancements
7+
8+
* Customizability
9+
* Introduced the new `return` action that allows action groups to return a value. See the [actions page](./docs/test/actions.md#return) for details.
10+
* Introduced new MFTF command that invokes `vendor/bin/codecept run`. See the [mftf page](./docs/commands/mftf.md#codeceptrun) for details.
11+
12+
* Usability
13+
* Introduced new action `pause`, to invoke codeception interactive pause for debugging during test execution. See the [Interactive Pause](./docs/interactive-pause.md) page for details.
14+
* Introduced a new `.env` configuration option `ENABLE_PAUSE`, to enable the new pause feature.
15+
16+
* Maintainability
17+
* Added a new static check that checks for the usage of the `pause` action. See the [command page](./docs/commands/mftf.md#static-checks) for details.
18+
19+
* Modularity
20+
* MFTF now supports the use of actions from multiple modules within suites.
21+
22+
* Traceability
23+
* A deprecation notice is now added at test execution time for deprecated metadata usage.
24+
25+
### Fixes
26+
27+
* Fixed issue with suite precondition failure for `createData` with required entity.
28+
29+
### GitHub Issues/Pull requests:
30+
31+
* [#547](https://github.com/magento/magento2-functional-testing-framework/pull/547) -- Fix invalid behavior of MAGENTO_BACKEND_BASE_URL
32+
* [#742](https://github.com/magento/magento2-functional-testing-framework/pull/742) -- Fix Waits In MagentoPwaWebDriver
33+
* [#750](https://github.com/magento/magento2-functional-testing-framework/pull/750) -- Docs: Outlining the difference between Allure severity levels
34+
* [#683](https://github.com/magento/magento2-functional-testing-framework/pull/683) -- Docs: Renamed sample test name with the correct one
35+
* [#691](https://github.com/magento/magento2-functional-testing-framework/pull/691) -- Docs: Branch name updates
36+
* [#678](https://github.com/magento/magento2-functional-testing-framework/pull/678) -- Docs: Command added to modify the web server rewrites configuration
37+
* [#745](https://github.com/magento/magento2-functional-testing-framework/pull/745) -- Docs: Remove invalid sample test name
38+
339
3.0.0
440
---------
541

‎composer.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
"name": "magento/magento2-functional-testing-framework",
33
"description": "Magento2 Functional Testing Framework",
44
"type": "library",
5-
"version": "3.0.0",
5+
"version": "3.1.0",
66
"license": "AGPL-3.0",
77
"keywords": ["magento", "automation", "functional", "testing"],
88
"config": {

‎composer.lock

Lines changed: 4 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎dev/tests/unit/Magento/FunctionalTestFramework/Test/Handlers/TestObjectHandlerTest.php

Lines changed: 2 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -260,25 +260,6 @@ public function testGetAllTestObjectsWithInvalidExtends()
260260
$toh->getAllObjects();
261261
}
262262

263-
/**
264-
* Function used to set mock for parser return and force init method to run between tests.
265-
*
266-
* @param array $data
267-
* @throws \Exception
268-
*/
269-
private function setMockParserOutput($data)
270-
{
271-
// clear test object handler value to inject parsed content
272-
$property = new \ReflectionProperty(TestObjectHandler::class, 'testObjectHandler');
273-
$property->setAccessible(true);
274-
$property->setValue(null);
275-
276-
$mockDataParser = AspectMock::double(TestDataParser::class, ['readTestData' => $data])->make();
277-
$instance = AspectMock::double(ObjectManager::class, ['create' => $mockDataParser])
278-
->make(); // bypass the private constructor
279-
AspectMock::double(ObjectManagerFactory::class, ['getObjectManager' => $instance]);
280-
}
281-
282263
/**
283264
* Validate test object when ENABLE_PAUSE is set to true
284265
*
@@ -299,7 +280,7 @@ public function testGetTestObjectWhenEnablePause()
299280

300281
$resolverMock = new MockModuleResolverBuilder();
301282
$resolverMock->setup();
302-
$this->setMockParserOutput($mockData);
283+
ObjectHandlerUtil::mockTestObjectHandlerWitData($mockData);
303284

304285
// run object handler method
305286
$toh = TestObjectHandler::getInstance();
@@ -325,7 +306,7 @@ public function testGetTestObjectWhenEnablePause()
325306
$expectedFailedActionObject2 = new ActionObject(
326307
'pauseWhenFailed',
327308
'pause',
328-
[]
309+
[ActionObject::PAUSE_ACTION_INTERNAL_ATTRIBUTE => true]
329310
);
330311

331312
$expectedBeforeHookObject = new TestHookObject(

‎docs/test/actions.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1251,8 +1251,9 @@ Attribute|Type|Use|Description
12511251
`stepKey`|string|required| A unique identifier of the action.
12521252

12531253
#### Example
1254+
12541255
```xml
1255-
<!-- Returns value of $grabInputName to the calling
1256+
<!-- Returns value of $grabInputName to the calling -->
12561257
<return value="{$grabInputName}" stepKey="returnInputName"/>
12571258
```
12581259

‎src/Magento/FunctionalTestingFramework/Console/BaseGenerateCommand.php

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ class BaseGenerateCommand extends Command
3333
const MFTF_NOTICES = "Placeholder text for MFTF notices\n";
3434
const CODECEPT_RUN = 'codecept:run';
3535
const CODECEPT_RUN_FUNCTIONAL = self::CODECEPT_RUN . ' functional ';
36+
const CODECEPT_RUN_OPTION_NO_EXIT = ' --no-exit ';
3637

3738
/**
3839
* Enable pause()

‎src/Magento/FunctionalTestingFramework/Console/RunManifestCommand.php

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -81,12 +81,17 @@ protected function execute(InputInterface $input, OutputInterface $output): int
8181
// Delete the Codeception failed file just in case it exists from any previous test runs
8282
$this->deleteFailedFile();
8383

84-
foreach ($manifestFile as $manifestLine) {
85-
if (empty($manifestLine)) {
84+
for ($line = 0; $line < count($manifestFile); $line++) {
85+
if (empty($manifestFile[$line])) {
8686
continue;
8787
}
8888

89-
$this->runManifestLine($manifestLine, $output);
89+
if ($line == count($manifestFile) - 1) {
90+
$this->runManifestLine($manifestFile[$line], $output, true);
91+
} else {
92+
$this->runManifestLine($manifestFile[$line], $output);
93+
}
94+
9095
$this->aggregateFailed();
9196
}
9297

@@ -103,18 +108,23 @@ protected function execute(InputInterface $input, OutputInterface $output): int
103108
*
104109
* @param string $manifestLine
105110
* @param OutputInterface $output
111+
* @param boolean $exit
106112
* @return void
107113
* @throws \Exception
108114
*
109115
* @SuppressWarnings(PHPMD.UnusedLocalVariable) Need this because of the unused $type variable in the closure
110116
*/
111-
private function runManifestLine(string $manifestLine, OutputInterface $output)
117+
private function runManifestLine($manifestLine, $output, $exit = false)
112118
{
113119
if (getenv('ENABLE_PAUSE') === 'true') {
114120
$codeceptionCommand = BaseGenerateCommand::CODECEPT_RUN_FUNCTIONAL
115-
. '--verbose --steps --debug ' . $manifestLine;
121+
. '--verbose --steps --debug ';
122+
if (!$exit) {
123+
$codeceptionCommand .= BaseGenerateCommand::CODECEPT_RUN_OPTION_NO_EXIT;
124+
}
125+
$codeceptionCommand .= $manifestLine;
116126
$input = new StringInput($codeceptionCommand);
117-
$command = $this->getApplication()->find('codecept:run');
127+
$command = $this->getApplication()->find(BaseGenerateCommand::CODECEPT_RUN);
118128
$subReturnCode = $command->run($input, $output);
119129
} else {
120130
$codeceptionCommand = realpath(PROJECT_ROOT . "/vendor/bin/codecept")

‎src/Magento/FunctionalTestingFramework/Console/RunTestCommand.php

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -135,8 +135,8 @@ private function runTests(array $tests, OutputInterface $output)
135135
TestGenerator::DEFAULT_DIR .
136136
DIRECTORY_SEPARATOR ;
137137

138-
foreach ($tests as $test) {
139-
$testName = $test . 'Cest.php';
138+
for ($i = 0; $i < count($tests); $i++) {
139+
$testName = $tests[$i] . 'Cest.php';
140140
if (!realpath($testsDirectory . $testName)) {
141141
throw new TestFrameworkException(
142142
$testName . " is not available under " . $testsDirectory
@@ -145,6 +145,9 @@ private function runTests(array $tests, OutputInterface $output)
145145

146146
if ($this->pauseEnabled()) {
147147
$fullCommand = $codeceptionCommand . $testsDirectory . $testName . ' --verbose --steps --debug';
148+
if ($i != count($tests) - 1) {
149+
$fullCommand .= self::CODECEPT_RUN_OPTION_NO_EXIT;
150+
}
148151
$this->returnCode = max($this->returnCode, $this->codeceptRunTest($fullCommand, $output));
149152
} else {
150153
$fullCommand = $codeceptionCommand . $testsDirectory . $testName . ' --verbose --steps';
@@ -169,10 +172,18 @@ private function runTestsInSuite(array $suitesConfig, OutputInterface $output)
169172
$codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept')
170173
. ' run functional --verbose --steps ';
171174
}
175+
176+
$count = count($suitesConfig);
177+
$index = 0;
172178
//for tests in suites, run them as a group to run before and after block
173179
foreach (array_keys($suitesConfig) as $suite) {
174180
$fullCommand = $codeceptionCommand . " -g {$suite}";
181+
182+
$index += 1;
175183
if ($this->pauseEnabled()) {
184+
if ($index != $count) {
185+
$fullCommand .= self::CODECEPT_RUN_OPTION_NO_EXIT;
186+
}
176187
$this->returnCode = max($this->returnCode, $this->codeceptRunTest($fullCommand, $output));
177188
} else {
178189
$this->returnCode = max($this->returnCode, $this->executeTestCommand($fullCommand, $output));

‎src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -116,13 +116,16 @@ protected function execute(InputInterface $input, OutputInterface $output): int
116116

117117
$testManifestList = $this->readTestManifestFile();
118118
$returnCode = 0;
119-
foreach ($testManifestList as $testCommand) {
119+
for ($i = 0; $i < count($testManifestList); $i++) {
120120
if ($this->pauseEnabled()) {
121-
$codeceptionCommand = self::CODECEPT_RUN_FUNCTIONAL . $testCommand . ' --debug';
121+
$codeceptionCommand = self::CODECEPT_RUN_FUNCTIONAL . $testManifestList[$i] . ' --debug ';
122+
if ($i != count($testManifestList) - 1) {
123+
$codeceptionCommand .= self::CODECEPT_RUN_OPTION_NO_EXIT;
124+
}
122125
$returnCode = $this->codeceptRunTest($codeceptionCommand, $output);
123126
} else {
124127
$codeceptionCommand = realpath(PROJECT_ROOT . '/vendor/bin/codecept') . ' run functional ';
125-
$codeceptionCommand .= $testCommand;
128+
$codeceptionCommand .= $testManifestList[$i];
126129

127130
$process = new Process($codeceptionCommand);
128131
$process->setWorkingDirectory(TESTS_BP);
@@ -142,6 +145,7 @@ function ($type, $buffer) use ($output) {
142145
);
143146
}
144147
}
148+
145149
foreach ($this->failedList as $test) {
146150
$this->writeFailedTestToFile($test, $this->testsFailedFile);
147151
}

‎src/Magento/FunctionalTestingFramework/Console/RunTestGroupCommand.php

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ protected function configure()
5252
* @throws \Exception
5353
*
5454
* @SuppressWarnings(PHPMD.UnusedLocalVariable)
55+
* @SuppressWarnings(PHPMD.CyclomaticComplexity)
5556
*/
5657
protected function execute(InputInterface $input, OutputInterface $output): int
5758
{
@@ -102,10 +103,13 @@ protected function execute(InputInterface $input, OutputInterface $output): int
102103

103104
$exitCode = -1;
104105
$returnCodes = [];
105-
foreach ($groups as $group) {
106-
$codeceptionCommandString = $commandString . " -g {$group}";
106+
for ($i = 0; $i < count($groups); $i++) {
107+
$codeceptionCommandString = $commandString . ' -g ' . $groups[$i];
107108

108109
if ($this->pauseEnabled()) {
110+
if ($i != count($groups) - 1) {
111+
$codeceptionCommandString .= self::CODECEPT_RUN_OPTION_NO_EXIT;
112+
}
109113
$returnCodes[] = $this->codeceptRunTest($codeceptionCommandString, $output);
110114
} else {
111115
$process = new Process($codeceptionCommandString);

‎src/Magento/FunctionalTestingFramework/Module/MagentoWebDriver.php

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,9 @@
5454
class MagentoWebDriver extends WebDriver
5555
{
5656
use AttachmentSupport;
57-
use Pause;
57+
use Pause {
58+
pause as codeceptPause;
59+
}
5860

5961
const MAGENTO_CRON_INTERVAL = 60;
6062
const MAGENTO_CRON_COMMAND = 'cron:run';
@@ -843,7 +845,7 @@ public function _failed(TestInterface $test, $fail)
843845
if ($this->pngReport === null && $this->htmlReport === null) {
844846
$this->saveScreenshot();
845847
if (getenv('ENABLE_PAUSE') === 'true') {
846-
$this->pause();
848+
$this->pause(true);
847849
}
848850
}
849851

@@ -1028,4 +1030,23 @@ public function switchToIFrame($locator = null)
10281030
$this->webDriver->switchTo()->frame($els[0]);
10291031
}
10301032
}
1033+
1034+
/**
1035+
* Invoke Codeption pause()
1036+
*
1037+
* @param boolean $pauseOnFail
1038+
* @return void
1039+
*/
1040+
public function pause($pauseOnFail = false)
1041+
{
1042+
if (!\Codeception\Util\Debug::isEnabled()) {
1043+
return;
1044+
}
1045+
1046+
if ($pauseOnFail) {
1047+
print(PHP_EOL . "Failure encountered. Pausing execution..." . PHP_EOL . PHP_EOL);
1048+
}
1049+
1050+
$this->codeceptPause();
1051+
}
10311052
}

‎src/Magento/FunctionalTestingFramework/Test/Objects/ActionObject.php

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ class ActionObject
7777
const ACTION_TYPE_COMMENT = 'comment';
7878
const ACTION_TYPE_HELPER = 'helper';
7979
const INVISIBLE_STEP_ACTIONS = ['retrieveEntityField', 'getSecret'];
80+
const PAUSE_ACTION_INTERNAL_ATTRIBUTE = 'pauseOnFail';
8081

8182
/**
8283
* The unique identifier for the action

‎src/Magento/FunctionalTestingFramework/Test/Util/TestHookObjectExtractor.php

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,11 @@ public function createDefaultFailedHook($parentName)
6666
{
6767
$defaultSteps['saveScreenshot'] = new ActionObject("saveScreenshot", "saveScreenshot", []);
6868
if (getenv('ENABLE_PAUSE') === 'true') {
69-
$defaultSteps['pauseWhenFailed'] = new ActionObject('pauseWhenFailed', 'pause', []);
69+
$defaultSteps['pauseWhenFailed'] = new ActionObject(
70+
'pauseWhenFailed',
71+
'pause',
72+
[ActionObject::PAUSE_ACTION_INTERNAL_ATTRIBUTE => true]
73+
);
7074
}
7175

7276
$hook = new TestHookObject(

‎src/Magento/FunctionalTestingFramework/Util/TestGenerator.php

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1441,6 +1441,16 @@ public function generateStepsPhp($actionObjects, $generationScope = TestGenerato
14411441

14421442
$testSteps .= $dateGenerateCode;
14431443
break;
1444+
case "pause":
1445+
$pauseAttr = $actionObject->getCustomActionAttributes(
1446+
ActionObject::PAUSE_ACTION_INTERNAL_ATTRIBUTE
1447+
);
1448+
if ($pauseAttr) {
1449+
$testSteps .= sprintf("\t\t$%s->%s(%s);", $actor, $actionObject->getType(), 'true');
1450+
} else {
1451+
$testSteps .= sprintf("\t\t$%s->%s();", $actor, $actionObject->getType());
1452+
}
1453+
break;
14441454
case "comment":
14451455
$input = $input === null ? strtr($value, ['$' => '\$', '{' => '\{', '}' => '\}']) : $input;
14461456
// Combining userInput from native XML comment and <comment/> action to fall-through 'default' case

0 commit comments

Comments
 (0)