From d79eb2a39225e8d1dabf1d6e551bdf7c74352403 Mon Sep 17 00:00:00 2001 From: Robert Oskamp Date: Mon, 20 Dec 2021 15:44:58 +0100 Subject: [PATCH] [ML] Add initial docs screenshot generation (#121495) This PR utilizes the functional test runner to walk through the UI and take a couple screenshots for use in the documentation. --- .../functional/services/common/screenshots.ts | 4 +- .../anomalies_table_columns.js | 2 +- .../anomalies_table/anomaly_details.js | 7 +- .../expandable_section/expandable_section.tsx | 8 +- .../exploration_query_bar.tsx | 2 +- .../application/explorer/anomalies_map.tsx | 2 +- x-pack/plugins/ml/readme.md | 14 + .../apps/ml/anomaly_detection/custom_urls.ts | 31 +- .../test/functional/page_objects/gis_page.ts | 14 + .../functional/services/ml/anomalies_table.ts | 24 ++ .../services/ml/anomaly_explorer.ts | 8 + x-pack/test/functional/services/ml/api.ts | 4 +- .../ml/data_frame_analytics_creation.ts | 32 +- .../ml/data_frame_analytics_results.ts | 54 +++- .../services/ml/data_visualizer_table.ts | 2 +- .../test/functional/services/ml/job_table.ts | 183 +++++------ .../services/ml/job_wizard_multi_metric.ts | 4 + .../test/functional/services/ml/navigation.ts | 7 + .../functional/services/ml/test_resources.ts | 34 ++ x-pack/test/screenshot_creation/apps/index.ts | 14 + .../ml_docs/anomaly_detection/custom_urls.ts | 101 ++++++ .../anomaly_detection/geographic_data.ts | 294 ++++++++++++++++++ .../apps/ml_docs/anomaly_detection/index.ts | 17 + .../anomaly_detection/mapping_anomalies.ts | 129 ++++++++ .../anomaly_detection/population_analysis.ts | 117 +++++++ .../data_frame_analytics/classification.ts | 163 ++++++++++ .../ml_docs/data_frame_analytics/index.ts | 16 + .../data_frame_analytics/outlier_detection.ts | 132 ++++++++ .../data_frame_analytics/regression.ts | 152 +++++++++ .../screenshot_creation/apps/ml_docs/index.ts | 35 +++ x-pack/test/screenshot_creation/config.ts | 24 ++ .../ftr_provider_context.d.ts | 13 + .../screenshot_creation/services/index.ts | 16 + .../services/ml_screenshots.ts | 25 ++ 34 files changed, 1555 insertions(+), 129 deletions(-) create mode 100644 x-pack/test/screenshot_creation/apps/index.ts create mode 100644 x-pack/test/screenshot_creation/apps/ml_docs/anomaly_detection/custom_urls.ts create mode 100644 x-pack/test/screenshot_creation/apps/ml_docs/anomaly_detection/geographic_data.ts create mode 100644 x-pack/test/screenshot_creation/apps/ml_docs/anomaly_detection/index.ts create mode 100644 x-pack/test/screenshot_creation/apps/ml_docs/anomaly_detection/mapping_anomalies.ts create mode 100644 x-pack/test/screenshot_creation/apps/ml_docs/anomaly_detection/population_analysis.ts create mode 100644 x-pack/test/screenshot_creation/apps/ml_docs/data_frame_analytics/classification.ts create mode 100644 x-pack/test/screenshot_creation/apps/ml_docs/data_frame_analytics/index.ts create mode 100644 x-pack/test/screenshot_creation/apps/ml_docs/data_frame_analytics/outlier_detection.ts create mode 100644 x-pack/test/screenshot_creation/apps/ml_docs/data_frame_analytics/regression.ts create mode 100644 x-pack/test/screenshot_creation/apps/ml_docs/index.ts create mode 100644 x-pack/test/screenshot_creation/config.ts create mode 100644 x-pack/test/screenshot_creation/ftr_provider_context.d.ts create mode 100644 x-pack/test/screenshot_creation/services/index.ts create mode 100644 x-pack/test/screenshot_creation/services/ml_screenshots.ts diff --git a/test/functional/services/common/screenshots.ts b/test/functional/services/common/screenshots.ts index 79ac5a1803545..0f2ab8e6edfbe 100644 --- a/test/functional/services/common/screenshots.ts +++ b/test/functional/services/common/screenshots.ts @@ -74,8 +74,8 @@ export class ScreenshotsService extends FtrService { } } - async take(name: string, el?: WebElementWrapper) { - const path = resolve(this.SESSION_DIRECTORY, `${name}.png`); + async take(name: string, el?: WebElementWrapper, subDirectories: string[] = []) { + const path = resolve(this.SESSION_DIRECTORY, ...subDirectories, `${name}.png`); await this.capture(path, el); this.failureMetadata.addScreenshot(name, path); } diff --git a/x-pack/plugins/ml/public/application/components/anomalies_table/anomalies_table_columns.js b/x-pack/plugins/ml/public/application/components/anomalies_table/anomalies_table_columns.js index 5add76d176da0..9e84dccf12b28 100644 --- a/x-pack/plugins/ml/public/application/components/anomalies_table/anomalies_table_columns.js +++ b/x-pack/plugins/ml/public/application/components/anomalies_table/anomalies_table_columns.js @@ -91,7 +91,7 @@ export function getColumns( }) } data-row-id={item.rowId} - data-test-subj="mlJobListRowDetailsToggle" + data-test-subj="mlAnomaliesListRowDetailsToggle" /> ), }, diff --git a/x-pack/plugins/ml/public/application/components/anomalies_table/anomaly_details.js b/x-pack/plugins/ml/public/application/components/anomalies_table/anomaly_details.js index bf3886d25bf52..d406784344f6d 100644 --- a/x-pack/plugins/ml/public/application/components/anomalies_table/anomaly_details.js +++ b/x-pack/plugins/ml/public/application/components/anomalies_table/anomaly_details.js @@ -313,7 +313,10 @@ export class AnomalyDetails extends Component { }), content: ( -
+
{this.renderDescription()} {this.renderDetails()} @@ -633,7 +636,7 @@ export class AnomalyDetails extends Component { ); } else { return ( -
+
{this.renderDescription()} {this.renderDetails()} diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/expandable_section/expandable_section.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/expandable_section/expandable_section.tsx index 6ce3c1f5a6548..7187eb22652e2 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/expandable_section/expandable_section.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/expandable_section/expandable_section.tsx @@ -84,6 +84,7 @@ export const ExpandableSection: FC = ({ iconType={isExpanded ? 'arrowUp' : 'arrowDown'} iconSide="right" flush="left" + data-test-subj={`mlDFExpandableSection-${dataTestId}-toggle-button`} > {title} @@ -126,7 +127,12 @@ export const ExpandableSection: FC = ({ )}
{isExpanded && ( -
{content}
+
+ {content} +
)} ); diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_query_bar/exploration_query_bar.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_query_bar/exploration_query_bar.tsx index 27eb06d7ecd41..f57ce0a8902f0 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_query_bar/exploration_query_bar.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_query_bar/exploration_query_bar.tsx @@ -192,7 +192,7 @@ export const ExplorationQueryBar: FC = ({ }) } disableAutoFocus={true} - dataTestSubj="transformQueryInput" + dataTestSubj="mlDFAnalyticsQueryInput" languageSwitcherPopoverAnchorPosition="rightDown" /> diff --git a/x-pack/plugins/ml/public/application/explorer/anomalies_map.tsx b/x-pack/plugins/ml/public/application/explorer/anomalies_map.tsx index 0eb6e356e1397..2bec87fafc17c 100644 --- a/x-pack/plugins/ml/public/application/explorer/anomalies_map.tsx +++ b/x-pack/plugins/ml/public/application/explorer/anomalies_map.tsx @@ -222,7 +222,7 @@ export const AnomaliesMap: FC = ({ anomalies, jobIds }) => { return ( <> - + { + const rowSubj = await this.getRowSubjByRowIndex(rowIndex); + if (!(await testSubjects.exists('mlAnomaliesListRowDetails'))) { + await testSubjects.click(`${rowSubj} > mlAnomaliesListRowDetailsToggle`); + await testSubjects.existOrFail('mlAnomaliesListRowDetails', { timeout: 1000 }); + } + }); + }, + + async ensureDetailsClosed(rowIndex: number) { + await retry.tryForTime(10 * 1000, async () => { + const rowSubj = await this.getRowSubjByRowIndex(rowIndex); + if (await testSubjects.exists('mlAnomaliesListRowDetails')) { + await testSubjects.click(`${rowSubj} > mlAnomaliesListRowDetailsToggle`); + await testSubjects.missingOrFail('mlAnomaliesListRowDetails', { timeout: 1000 }); + } + }); + }, + + async scrollTableIntoView() { + await testSubjects.scrollIntoView('mlAnomaliesTable'); + }, }; } diff --git a/x-pack/test/functional/services/ml/anomaly_explorer.ts b/x-pack/test/functional/services/ml/anomaly_explorer.ts index 22032799020ad..9c48ed24c78a3 100644 --- a/x-pack/test/functional/services/ml/anomaly_explorer.ts +++ b/x-pack/test/functional/services/ml/anomaly_explorer.ts @@ -174,5 +174,13 @@ export function MachineLearningAnomalyExplorerProvider({ `Expect ${expectedChartsCount} charts to appear, got ${actualChartsCount}` ); }, + + async scrollChartsContainerIntoView() { + await testSubjects.scrollIntoView('mlExplorerChartsContainer'); + }, + + async scrollMapContainerIntoView() { + await testSubjects.scrollIntoView('mlAnomaliesMapContainer'); + }, }; } diff --git a/x-pack/test/functional/services/ml/api.ts b/x-pack/test/functional/services/ml/api.ts index a9a44f58a84df..ebe7f7e84d158 100644 --- a/x-pack/test/functional/services/ml/api.ts +++ b/x-pack/test/functional/services/ml/api.ts @@ -936,11 +936,11 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) { log.debug('> DFA job started.'); }, - async createAndRunDFAJob(dfaConfig: DataFrameAnalyticsConfig) { + async createAndRunDFAJob(dfaConfig: DataFrameAnalyticsConfig, timeout?: number) { await this.createDataFrameAnalyticsJob(dfaConfig); await this.runDFAJob(dfaConfig.id); await this.waitForDFAJobTrainingRecordCountToBePositive(dfaConfig.id); - await this.waitForAnalyticsState(dfaConfig.id, DATA_FRAME_TASK_STATE.STOPPED); + await this.waitForAnalyticsState(dfaConfig.id, DATA_FRAME_TASK_STATE.STOPPED, timeout); }, async updateJobSpaces( diff --git a/x-pack/test/functional/services/ml/data_frame_analytics_creation.ts b/x-pack/test/functional/services/ml/data_frame_analytics_creation.ts index 1b28589fc21d3..3bb0f6d8169ee 100644 --- a/x-pack/test/functional/services/ml/data_frame_analytics_creation.ts +++ b/x-pack/test/functional/services/ml/data_frame_analytics_creation.ts @@ -23,6 +23,8 @@ export function MachineLearningDataFrameAnalyticsCreationProvider( mlApi: MlApi ) { const headerPage = getPageObject('header'); + const commonPage = getPageObject('common'); + const testSubjects = getService('testSubjects'); const comboBox = getService('comboBox'); const retry = getService('retry'); @@ -33,6 +35,10 @@ export function MachineLearningDataFrameAnalyticsCreationProvider( await testSubjects.existOrFail('mlAnalyticsCreateJobWizardJobTypeSelect'); }, + async scrollJobTypeSelectionIntoView() { + await testSubjects.scrollIntoView('mlAnalyticsCreateJobWizardJobTypeSelect'); + }, + async assertJobTypeSelection(jobTypeAttribute: string) { await retry.tryForTime(5000, async () => { await testSubjects.existOrFail(`${jobTypeAttribute} selectedJobType`); @@ -324,16 +330,24 @@ export function MachineLearningDataFrameAnalyticsCreationProvider( await this.assertDependentVariableSelection([dependentVariable]); }, - async assertScatterplotMatrix(expectedValue: CanvasElementColorStats) { + async assertScatterplotMatrixLoaded() { await testSubjects.existOrFail( 'mlAnalyticsCreateJobWizardScatterplotMatrixPanel > mlScatterplotMatrix loaded', { timeout: 5000, } ); + }, + + async scrollScatterplotMatrixIntoView() { await testSubjects.scrollIntoView( 'mlAnalyticsCreateJobWizardScatterplotMatrixPanel > mlScatterplotMatrix loaded' ); + }, + + async assertScatterplotMatrix(expectedValue: CanvasElementColorStats) { + await this.assertScatterplotMatrixLoaded(); + await this.scrollScatterplotMatrixIntoView(); await mlCommonUI.assertColorsInCanvasElement( 'mlAnalyticsCreateJobWizardScatterplotMatrixPanel', expectedValue, @@ -672,5 +686,21 @@ export function MachineLearningDataFrameAnalyticsCreationProvider( await testSubjects.click('analyticsWizardCardManagement'); await testSubjects.existOrFail('mlPageDataFrameAnalytics'); }, + + async assertQueryBarValue(expectedValue: string) { + const actualQuery = await testSubjects.getAttribute('mlDFAnalyticsQueryInput', 'value'); + expect(actualQuery).to.eql( + expectedValue, + `Query should be '${expectedValue}' (got '${actualQuery}')` + ); + }, + + async setQueryBarValue(query: string) { + await mlCommonUI.setValueWithChecks('mlDFAnalyticsQueryInput', query, { + clearWithKeyboard: true, + }); + await commonPage.pressEnterKey(); + await this.assertQueryBarValue(query); + }, }; } diff --git a/x-pack/test/functional/services/ml/data_frame_analytics_results.ts b/x-pack/test/functional/services/ml/data_frame_analytics_results.ts index cf34a1372157c..dc7ccc9130fab 100644 --- a/x-pack/test/functional/services/ml/data_frame_analytics_results.ts +++ b/x-pack/test/functional/services/ml/data_frame_analytics_results.ts @@ -33,6 +33,10 @@ export function MachineLearningDataFrameAnalyticsResultsProvider( await testSubjects.existOrFail('mlDFAnalyticsExplorationTablePanel'); }, + async scrollRocCurveChartIntoView() { + await testSubjects.scrollIntoView('mlDFAnalyticsClassificationExplorationRocCurveChart'); + }, + async assertClassificationEvaluatePanelElementsExists() { await testSubjects.existOrFail('mlDFExpandableSection-ClassificationEvaluation'); await testSubjects.existOrFail('mlDFAnalyticsClassificationExplorationConfusionMatrix'); @@ -125,11 +129,15 @@ export function MachineLearningDataFrameAnalyticsResultsProvider( }); }, - async assertScatterplotMatrix(expectedValue: CanvasElementColorStats) { + async assertScatterplotMatrixLoaded() { await testSubjects.existOrFail('mlDFExpandableSection-splom > mlScatterplotMatrix loaded', { timeout: 5000, }); - await testSubjects.scrollIntoView('mlDFExpandableSection-splom > mlScatterplotMatrix loaded'); + }, + + async assertScatterplotMatrix(expectedValue: CanvasElementColorStats) { + await this.assertScatterplotMatrixLoaded(); + await this.scrollScatterplotMatrixIntoView(); await mlCommonUI.assertColorsInCanvasElement( 'mlDFExpandableSection-splom', expectedValue, @@ -242,5 +250,47 @@ export function MachineLearningDataFrameAnalyticsResultsProvider( async scrollResultsIntoView() { await this.scrollContentSectionIntoView('results'); }, + + async expandContentSection(sectionId: string, shouldExpand: boolean) { + const contentSubj = `mlDFExpandableSection-${sectionId}-content`; + const expandableContentExists = await testSubjects.exists(contentSubj, { timeout: 1000 }); + + if (expandableContentExists !== shouldExpand) { + await retry.tryForTime(5 * 1000, async () => { + await testSubjects.clickWhenNotDisabled( + `mlDFExpandableSection-${sectionId}-toggle-button` + ); + if (shouldExpand) { + await testSubjects.existOrFail(contentSubj, { timeout: 1000 }); + } else { + await testSubjects.missingOrFail(contentSubj, { timeout: 1000 }); + } + }); + } + }, + + async expandAnalysisSection(shouldExpand: boolean) { + await this.expandContentSection('analysis', shouldExpand); + }, + + async expandRegressionEvaluationSection(shouldExpand: boolean) { + await this.expandContentSection('RegressionEvaluation', shouldExpand); + }, + + async expandClassificationEvaluationSection(shouldExpand: boolean) { + await this.expandContentSection('ClassificationEvaluation', shouldExpand); + }, + + async expandFeatureImportanceSection(shouldExpand: boolean) { + await this.expandContentSection('FeatureImportanceSummary', shouldExpand); + }, + + async expandScatterplotMatrixSection(shouldExpand: boolean) { + await this.expandContentSection('splom', shouldExpand); + }, + + async expandResultsSection(shouldExpand: boolean) { + await this.expandContentSection('results', shouldExpand); + }, }; } diff --git a/x-pack/test/functional/services/ml/data_visualizer_table.ts b/x-pack/test/functional/services/ml/data_visualizer_table.ts index 860f2bd86bec7..24563fe05f6ff 100644 --- a/x-pack/test/functional/services/ml/data_visualizer_table.ts +++ b/x-pack/test/functional/services/ml/data_visualizer_table.ts @@ -295,7 +295,7 @@ export function MachineLearningDataVisualizerTableProvider( } public async setSampleSizeInputValue( - sampleSize: number, + sampleSize: number | 'all', fieldName: string, docCountFormatted: string ) { diff --git a/x-pack/test/functional/services/ml/job_table.ts b/x-pack/test/functional/services/ml/job_table.ts index e2d50c52c55ba..ddf5cab918f05 100644 --- a/x-pack/test/functional/services/ml/job_table.ts +++ b/x-pack/test/functional/services/ml/job_table.ts @@ -20,6 +20,27 @@ import { export type MlADJobTable = ProvidedType; +export interface DiscoverUrlConfig { + label: string; + indexPattern: string; + queryEntityFieldNames: string[]; + timeRange: TimeRangeType; + timeRangeInterval?: string; +} + +export interface DashboardUrlConfig { + label: string; + dashboardName: string; + queryEntityFieldNames: string[]; + timeRange: TimeRangeType; + timeRangeInterval?: string; +} + +export interface OtherUrlConfig { + label: string; + url: string; +} + export function MachineLearningJobTableProvider( { getService }: FtrProviderContext, mlCommonUI: MlCommonUI, @@ -564,121 +585,103 @@ export function MachineLearningJobTableProvider( await testSubjects.existOrFail('mlJobCustomUrlForm'); } - public async addDiscoverCustomUrl( - jobId: string, - customUrl: { - label: string; - indexPattern: string; - queryEntityFieldNames: string[]; - timeRange: TimeRangeType; - timeRangeInterval?: string; + public async getExistingCustomUrlCount(): Promise { + const existingCustomUrls = await testSubjects.findAll('mlJobEditCustomUrlItemLabel'); + return existingCustomUrls.length; + } + + public async saveCustomUrl(expectedLabel: string, expectedIndex: number) { + await retry.tryForTime(5000, async () => { + await testSubjects.click('mlJobAddCustomUrl'); + await customUrls.assertCustomUrlLabel(expectedIndex, expectedLabel); + }); + } + + public async fillInDiscoverUrlForm(customUrl: DiscoverUrlConfig) { + await this.clickOpenCustomUrlEditor(); + await customUrls.setCustomUrlLabel(customUrl.label); + await mlCommonUI.selectRadioGroupValue( + `mlJobCustomUrlLinkToTypeInput`, + URL_TYPE.KIBANA_DISCOVER + ); + await mlCommonUI.selectSelectValueByVisibleText( + 'mlJobCustomUrlDiscoverIndexPatternInput', + customUrl.indexPattern + ); + await customUrls.setCustomUrlQueryEntityFieldNames(customUrl.queryEntityFieldNames); + await mlCommonUI.selectSelectValueByVisibleText( + 'mlJobCustomUrlTimeRangeInput', + customUrl.timeRange + ); + if (customUrl.timeRange === TIME_RANGE_TYPE.INTERVAL) { + await customUrls.setCustomUrlTimeRangeInterval(customUrl.timeRangeInterval!); } - ) { + } + + public async fillInDashboardUrlForm(customUrl: DashboardUrlConfig) { + await this.clickOpenCustomUrlEditor(); + await customUrls.setCustomUrlLabel(customUrl.label); + await mlCommonUI.selectRadioGroupValue( + `mlJobCustomUrlLinkToTypeInput`, + URL_TYPE.KIBANA_DASHBOARD + ); + await mlCommonUI.selectSelectValueByVisibleText( + 'mlJobCustomUrlDashboardNameInput', + customUrl.dashboardName + ); + await customUrls.setCustomUrlQueryEntityFieldNames(customUrl.queryEntityFieldNames); + await mlCommonUI.selectSelectValueByVisibleText( + 'mlJobCustomUrlTimeRangeInput', + customUrl.timeRange + ); + if (customUrl.timeRange === TIME_RANGE_TYPE.INTERVAL) { + await customUrls.setCustomUrlTimeRangeInterval(customUrl.timeRangeInterval!); + } + } + + public async fillInOtherUrlForm(customUrl: OtherUrlConfig) { + await this.clickOpenCustomUrlEditor(); + await customUrls.setCustomUrlLabel(customUrl.label); + await mlCommonUI.selectRadioGroupValue(`mlJobCustomUrlLinkToTypeInput`, URL_TYPE.OTHER); + await customUrls.setCustomUrlOtherTypeUrl(customUrl.url); + } + + public async addDiscoverCustomUrl(jobId: string, customUrl: DiscoverUrlConfig) { await retry.tryForTime(30 * 1000, async () => { await this.closeEditJobFlyout(); await this.openEditCustomUrlsForJobTab(jobId); + const existingCustomUrlCount = await this.getExistingCustomUrlCount(); - const existingCustomUrls = await testSubjects.findAll('mlJobEditCustomUrlItemLabel'); - - // Fill-in the form - await this.clickOpenCustomUrlEditor(); - await customUrls.setCustomUrlLabel(customUrl.label); - await mlCommonUI.selectRadioGroupValue( - `mlJobCustomUrlLinkToTypeInput`, - URL_TYPE.KIBANA_DISCOVER - ); - await mlCommonUI.selectSelectValueByVisibleText( - 'mlJobCustomUrlDiscoverIndexPatternInput', - customUrl.indexPattern - ); - await customUrls.setCustomUrlQueryEntityFieldNames(customUrl.queryEntityFieldNames); - await mlCommonUI.selectSelectValueByVisibleText( - 'mlJobCustomUrlTimeRangeInput', - customUrl.timeRange - ); - if (customUrl.timeRange === TIME_RANGE_TYPE.INTERVAL) { - await customUrls.setCustomUrlTimeRangeInterval(customUrl.timeRangeInterval!); - } - - // Save custom URL - await retry.tryForTime(5000, async () => { - await testSubjects.click('mlJobAddCustomUrl'); - const expectedIndex = existingCustomUrls.length; - await customUrls.assertCustomUrlLabel(expectedIndex, customUrl.label); - }); + await this.fillInDiscoverUrlForm(customUrl); + await this.saveCustomUrl(customUrl.label, existingCustomUrlCount); }); // Save the job await this.saveEditJobFlyoutChanges(); } - public async addDashboardCustomUrl( - jobId: string, - customUrl: { - label: string; - dashboardName: string; - queryEntityFieldNames: string[]; - timeRange: TimeRangeType; - timeRangeInterval?: string; - } - ) { + public async addDashboardCustomUrl(jobId: string, customUrl: DashboardUrlConfig) { await retry.tryForTime(30 * 1000, async () => { await this.closeEditJobFlyout(); await this.openEditCustomUrlsForJobTab(jobId); + const existingCustomUrlCount = await this.getExistingCustomUrlCount(); - const existingCustomUrls = await testSubjects.findAll('mlJobEditCustomUrlItemLabel'); - - // Fill-in the form - await this.clickOpenCustomUrlEditor(); - await customUrls.setCustomUrlLabel(customUrl.label); - await mlCommonUI.selectRadioGroupValue( - `mlJobCustomUrlLinkToTypeInput`, - URL_TYPE.KIBANA_DASHBOARD - ); - await mlCommonUI.selectSelectValueByVisibleText( - 'mlJobCustomUrlDashboardNameInput', - customUrl.dashboardName - ); - await customUrls.setCustomUrlQueryEntityFieldNames(customUrl.queryEntityFieldNames); - await mlCommonUI.selectSelectValueByVisibleText( - 'mlJobCustomUrlTimeRangeInput', - customUrl.timeRange - ); - if (customUrl.timeRange === TIME_RANGE_TYPE.INTERVAL) { - await customUrls.setCustomUrlTimeRangeInterval(customUrl.timeRangeInterval!); - } - - // Save custom URL - await retry.tryForTime(5000, async () => { - await testSubjects.click('mlJobAddCustomUrl'); - const expectedIndex = existingCustomUrls.length; - await customUrls.assertCustomUrlLabel(expectedIndex, customUrl.label); - }); + await this.fillInDashboardUrlForm(customUrl); + await this.saveCustomUrl(customUrl.label, existingCustomUrlCount); }); // Save the job await this.saveEditJobFlyoutChanges(); } - public async addOtherTypeCustomUrl(jobId: string, customUrl: { label: string; url: string }) { + public async addOtherTypeCustomUrl(jobId: string, customUrl: OtherUrlConfig) { await retry.tryForTime(30 * 1000, async () => { await this.closeEditJobFlyout(); await this.openEditCustomUrlsForJobTab(jobId); + const existingCustomUrlCount = await this.getExistingCustomUrlCount(); - const existingCustomUrls = await testSubjects.findAll('mlJobEditCustomUrlItemLabel'); - - // Fill-in the form - await this.clickOpenCustomUrlEditor(); - await customUrls.setCustomUrlLabel(customUrl.label); - await mlCommonUI.selectRadioGroupValue(`mlJobCustomUrlLinkToTypeInput`, URL_TYPE.OTHER); - await customUrls.setCustomUrlOtherTypeUrl(customUrl.url); - - // Save custom URL - await retry.tryForTime(5000, async () => { - await testSubjects.click('mlJobAddCustomUrl'); - const expectedIndex = existingCustomUrls.length; - await customUrls.assertCustomUrlLabel(expectedIndex, customUrl.label); - }); + await this.fillInOtherUrlForm(customUrl); + await this.saveCustomUrl(customUrl.label, existingCustomUrlCount); }); // Save the job diff --git a/x-pack/test/functional/services/ml/job_wizard_multi_metric.ts b/x-pack/test/functional/services/ml/job_wizard_multi_metric.ts index 69fd1a6a67ba7..2d25144142baf 100644 --- a/x-pack/test/functional/services/ml/job_wizard_multi_metric.ts +++ b/x-pack/test/functional/services/ml/job_wizard_multi_metric.ts @@ -33,6 +33,10 @@ export function MachineLearningJobWizardMultiMetricProvider({ getService }: FtrP await this.assertSplitFieldSelection([identifier]); }, + async scrollSplitFieldIntoView() { + await testSubjects.scrollIntoView('mlMultiMetricSplitFieldSelect'); + }, + async assertDetectorSplitExists(splitField: string) { await testSubjects.existOrFail(`mlDataSplit > mlDataSplitTitle ${splitField}`); await testSubjects.existOrFail(`mlDataSplit > mlSplitCard front`); diff --git a/x-pack/test/functional/services/ml/navigation.ts b/x-pack/test/functional/services/ml/navigation.ts index 0027405e4bf39..1da98dcdd2399 100644 --- a/x-pack/test/functional/services/ml/navigation.ts +++ b/x-pack/test/functional/services/ml/navigation.ts @@ -223,6 +223,13 @@ export function MachineLearningNavigationProvider({ await testSubjects.existOrFail('collapsibleNav'); }, + async closeKibanaNav() { + if (await testSubjects.exists('collapsibleNav')) { + await testSubjects.click('toggleNavButton'); + } + await testSubjects.missingOrFail('collapsibleNav'); + }, + async assertKibanaNavMLEntryExists() { const navArea = await testSubjects.find('collapsibleNav'); const mlNavLink = await navArea.findAllByCssSelector('[title="Machine Learning"]'); diff --git a/x-pack/test/functional/services/ml/test_resources.ts b/x-pack/test/functional/services/ml/test_resources.ts index b9e80c3ebf5e9..afb808b0f0bb1 100644 --- a/x-pack/test/functional/services/ml/test_resources.ts +++ b/x-pack/test/functional/services/ml/test_resources.ts @@ -537,5 +537,39 @@ export function MachineLearningTestResourcesProvider({ getService }: FtrProvider async clearAdvancedSettingProperty(propertyName: string) { await kibanaServer.uiSettings.unset(propertyName); }, + + async installKibanaSampleData(sampleDataId: 'ecommerce' | 'flights' | 'logs') { + log.debug(`Installing Kibana sample data '${sampleDataId}'`); + + await supertest + .post(`/api/sample_data/${sampleDataId}`) + .set(COMMON_REQUEST_HEADERS) + .expect(200); + + log.debug(` > Installed`); + }, + + async removeKibanaSampleData(sampleDataId: 'ecommerce' | 'flights' | 'logs') { + log.debug(`Removing Kibana sample data '${sampleDataId}'`); + + await supertest + .delete(`/api/sample_data/${sampleDataId}`) + .set(COMMON_REQUEST_HEADERS) + .expect(204); // No Content + + log.debug(` > Removed`); + }, + + async installAllKibanaSampleData() { + await this.installKibanaSampleData('ecommerce'); + await this.installKibanaSampleData('flights'); + await this.installKibanaSampleData('logs'); + }, + + async removeAllKibanaSampleData() { + await this.removeKibanaSampleData('ecommerce'); + await this.removeKibanaSampleData('flights'); + await this.removeKibanaSampleData('logs'); + }, }; } diff --git a/x-pack/test/screenshot_creation/apps/index.ts b/x-pack/test/screenshot_creation/apps/index.ts new file mode 100644 index 0000000000000..b02cf516a0088 --- /dev/null +++ b/x-pack/test/screenshot_creation/apps/index.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../ftr_provider_context'; + +export default function ({ loadTestFile }: FtrProviderContext) { + describe('apps', function () { + loadTestFile(require.resolve('./ml_docs')); + }); +} diff --git a/x-pack/test/screenshot_creation/apps/ml_docs/anomaly_detection/custom_urls.ts b/x-pack/test/screenshot_creation/apps/ml_docs/anomaly_detection/custom_urls.ts new file mode 100644 index 0000000000000..a823bff61dbdf --- /dev/null +++ b/x-pack/test/screenshot_creation/apps/ml_docs/anomaly_detection/custom_urls.ts @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../../ftr_provider_context'; +import { Job, Datafeed } from '../../../../../plugins/ml/common/types/anomaly_detection_jobs'; +import { TIME_RANGE_TYPE } from '../../../../../plugins/ml/public/application/jobs/components/custom_url_editor/constants'; + +import { ECOMMERCE_INDEX_PATTERN } from '../index'; + +export default function ({ getService }: FtrProviderContext) { + const ml = getService('ml'); + const mlScreenshots = getService('mlScreenshots'); + + const screenshotDirectories = ['ml_docs', 'anomaly_detection']; + + const ecommerceJobConfig = { + job_id: `ecommerce-custom-url`, + analysis_config: { + bucket_span: '2h', + influencers: ['geoip.country_iso_code', 'day_of_week', 'category.keyword', 'user'], + detectors: [ + { + detector_description: 'mean("products.base_price") over "customer_full_name.keyword"', + function: 'mean', + field_name: 'products.base_price', + over_field_name: 'customer_full_name.keyword', + }, + ], + }, + data_description: { time_field: 'order_date' }, + }; + + const ecommerceDatafeedConfig = { + datafeed_id: 'datafeed-ecommerce-custom-url', + indices: [ECOMMERCE_INDEX_PATTERN], + job_id: 'ecommerce-custom-url', + query: { bool: { must: [{ match_all: {} }] } }, + }; + + const testDashboardCustomUrl = { + label: 'Data dashboard', + dashboardName: '[eCommerce] Revenue Dashboard', + queryEntityFieldNames: ['customer_full_name.keyword'], + timeRange: TIME_RANGE_TYPE.AUTO, + }; + + describe('custom urls', function () { + before(async () => { + await ml.api.createAndRunAnomalyDetectionLookbackJob( + ecommerceJobConfig as Job, + ecommerceDatafeedConfig as Datafeed + ); + }); + + after(async () => { + await ml.api.deleteAnomalyDetectionJobES(ecommerceJobConfig.job_id); + await ml.api.cleanMlIndices(); + }); + + it('custom url config screenshot', async () => { + await ml.testExecution.logTestStep('navigate to job list'); + await ml.navigation.navigateToMl(); + await ml.navigation.navigateToJobManagement(); + + await ml.testExecution.logTestStep( + 'fill in the dashboard custom url form and take screenshot' + ); + await ml.jobTable.closeEditJobFlyout(); + await ml.jobTable.openEditCustomUrlsForJobTab(ecommerceJobConfig.job_id); + const existingCustomUrlCount = await ml.jobTable.getExistingCustomUrlCount(); + await ml.jobTable.fillInDashboardUrlForm(testDashboardCustomUrl); + await mlScreenshots.takeScreenshot('ml-customurl-edit', screenshotDirectories); + + await ml.testExecution.logTestStep('add the custom url and save the job'); + await ml.jobTable.saveCustomUrl(testDashboardCustomUrl.label, existingCustomUrlCount); + await ml.jobTable.saveEditJobFlyoutChanges(); + }); + + it('anomaly list screenshot', async () => { + await ml.testExecution.logTestStep('navigate to job list'); + await ml.navigation.navigateToMl(); + await ml.navigation.navigateToJobManagement(); + + await ml.testExecution.logTestStep('open job in anomaly explorer'); + await ml.jobTable.waitForJobsToLoad(); + await ml.jobTable.filterWithSearchString(ecommerceJobConfig.job_id, 1); + await ml.jobTable.clickOpenJobInAnomalyExplorerButton(ecommerceJobConfig.job_id); + await ml.commonUI.waitForMlLoadingIndicatorToDisappear(); + + await ml.testExecution.logTestStep('open anomaly list actions and take screenshot'); + await ml.anomaliesTable.scrollTableIntoView(); + await ml.anomaliesTable.ensureAnomalyActionsMenuOpen(0); + + await mlScreenshots.takeScreenshot('ml-population-results', screenshotDirectories); + }); + }); +} diff --git a/x-pack/test/screenshot_creation/apps/ml_docs/anomaly_detection/geographic_data.ts b/x-pack/test/screenshot_creation/apps/ml_docs/anomaly_detection/geographic_data.ts new file mode 100644 index 0000000000000..24a92612a7595 --- /dev/null +++ b/x-pack/test/screenshot_creation/apps/ml_docs/anomaly_detection/geographic_data.ts @@ -0,0 +1,294 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../../ftr_provider_context'; +import { Job, Datafeed } from '../../../../../plugins/ml/common/types/anomaly_detection_jobs'; +import { ML_JOB_FIELD_TYPES } from '../../../../../plugins/ml/common/constants/field_types'; + +import { ECOMMERCE_INDEX_PATTERN, LOGS_INDEX_PATTERN } from '../index'; + +export default function ({ getPageObject, getService }: FtrProviderContext) { + const elasticChart = getService('elasticChart'); + const maps = getPageObject('maps'); + const ml = getService('ml'); + const mlScreenshots = getService('mlScreenshots'); + const renderable = getService('renderable'); + + const screenshotDirectories = ['ml_docs', 'anomaly_detection']; + + const ecommerceGeoJobConfig = { + job_id: `ecommerce-geo`, + analysis_config: { + bucket_span: '15m', + influencers: ['geoip.country_iso_code', 'day_of_week', 'category.keyword', 'user'], + detectors: [ + { + detector_description: 'Unusual coordinates by user', + function: 'lat_long', + field_name: 'geoip.location', + by_field_name: 'user', + }, + ], + }, + data_description: { time_field: 'order_date' }, + }; + + const ecommerceGeoDatafeedConfig = { + datafeed_id: 'datafeed-ecommerce-geo', + indices: [ECOMMERCE_INDEX_PATTERN], + job_id: 'ecommerce-geo', + query: { bool: { must: [{ match_all: {} }] } }, + }; + + const weblogGeoJobConfig = { + job_id: `weblogs-geo`, + analysis_config: { + bucket_span: '15m', + influencers: ['geo.src', 'extension.keyword', 'geo.dest'], + detectors: [ + { + detector_description: 'Unusual coordinates', + function: 'lat_long', + field_name: 'geo.coordinates', + }, + { + function: 'high_sum', + field_name: 'bytes', + }, + ], + }, + data_description: { time_field: 'timestamp', time_format: 'epoch_ms' }, + }; + + const weblogGeoDatafeedConfig = { + datafeed_id: 'datafeed-weblogs-geo', + indices: [LOGS_INDEX_PATTERN], + job_id: 'weblogs-geo', + query: { bool: { must: [{ match_all: {} }] } }, + }; + + const cellSize = 15; + const overallSwimLaneTestSubj = 'mlAnomalyExplorerSwimlaneOverall'; + + describe('geographic data', function () { + before(async () => { + await ml.api.createAndRunAnomalyDetectionLookbackJob( + ecommerceGeoJobConfig as Job, + ecommerceGeoDatafeedConfig as Datafeed + ); + await ml.api.createAndRunAnomalyDetectionLookbackJob( + weblogGeoJobConfig as Job, + weblogGeoDatafeedConfig as Datafeed + ); + }); + + after(async () => { + await elasticChart.setNewChartUiDebugFlag(false); + await ml.api.deleteAnomalyDetectionJobES(ecommerceGeoJobConfig.job_id); + await ml.api.deleteAnomalyDetectionJobES(weblogGeoJobConfig.job_id); + await ml.api.cleanMlIndices(); + }); + + it('data visualizer screenshot', async () => { + await ml.testExecution.logTestStep('open index in data visualizer'); + await ml.navigation.navigateToMl(); + await ml.navigation.navigateToDataVisualizer(); + await ml.dataVisualizer.navigateToIndexPatternSelection(); + await ml.jobSourceSelection.selectSourceForIndexBasedDataVisualizer(LOGS_INDEX_PATTERN); + + await ml.testExecution.logTestStep('set data visualizer options'); + await ml.dataVisualizerIndexBased.assertTimeRangeSelectorSectionExists(); + await ml.dataVisualizerIndexBased.clickUseFullDataButton('14,074'); + await ml.dataVisualizerTable.setSampleSizeInputValue( + 'all', + 'geo.coordinates', + '14074 (100%)' + ); + await ml.dataVisualizerTable.setFieldTypeFilter([ML_JOB_FIELD_TYPES.GEO_POINT]); + + await ml.testExecution.logTestStep('set maps options and take screenshot'); + await ml.dataVisualizerTable.ensureDetailsOpen('geo.coordinates'); + await renderable.waitForRender(); + + // setView only works with displayed legend + await maps.openLegend(); + await maps.setView(44.1, -68.9, 4.5); + await maps.closeLegend(); + + await mlScreenshots.takeScreenshot('weblogs-data-visualizer-geopoint', screenshotDirectories); + }); + + it('ecommerce wizard screenshot', async () => { + await ml.testExecution.logTestStep('navigate to job list'); + await ml.navigation.navigateToMl(); + await ml.navigation.navigateToJobManagement(); + + await ml.testExecution.logTestStep('load the advanced wizard'); + await ml.jobManagement.navigateToNewJobSourceSelection(); + await ml.jobSourceSelection.selectSourceForAnomalyDetectionJob(ECOMMERCE_INDEX_PATTERN); + await ml.jobTypeSelection.selectAdvancedJob(); + + await ml.testExecution.logTestStep('continue to the pick fields step'); + await ml.jobWizardCommon.assertConfigureDatafeedSectionExists(); + await ml.jobWizardCommon.advanceToPickFieldsSection(); + + await ml.testExecution.logTestStep('add detector'); + await ml.jobWizardAdvanced.openCreateDetectorModal(); + await ml.jobWizardAdvanced.selectDetectorFunction('lat_long'); + await ml.jobWizardAdvanced.selectDetectorField('geoip.location'); + await ml.jobWizardAdvanced.selectDetectorByField('user'); + await ml.jobWizardAdvanced.confirmAddDetectorModal(); + + await ml.testExecution.logTestStep('set the bucket span'); + await ml.jobWizardCommon.assertBucketSpanInputExists(); + await ml.jobWizardCommon.setBucketSpan('15m'); + + await ml.testExecution.logTestStep('set influencers'); + await ml.jobWizardCommon.assertInfluencerInputExists(); + await ml.jobWizardCommon.assertInfluencerSelection([]); + for (const influencer of ['geoip.country_iso_code', 'day_of_week', 'category.keyword']) { + await ml.jobWizardCommon.addInfluencer(influencer); + } + + await ml.testExecution.logTestStep('set the model memory limit'); + await ml.jobWizardCommon.assertModelMemoryLimitInputExists({ + withAdvancedSection: false, + }); + await ml.jobWizardCommon.setModelMemoryLimit('12MB', { + withAdvancedSection: false, + }); + + await ml.testExecution.logTestStep('take screenshot'); + await mlScreenshots.removeFocusFromElement(); + await mlScreenshots.takeScreenshot( + 'ecommerce-advanced-wizard-geopoint', + screenshotDirectories + ); + }); + + it('weblogs wizard screenshot', async () => { + await ml.testExecution.logTestStep('navigate to job list'); + await ml.navigation.navigateToMl(); + await ml.navigation.navigateToJobManagement(); + + await ml.testExecution.logTestStep('load the advanced wizard'); + await ml.jobManagement.navigateToNewJobSourceSelection(); + await ml.jobSourceSelection.selectSourceForAnomalyDetectionJob(LOGS_INDEX_PATTERN); + await ml.jobTypeSelection.selectAdvancedJob(); + + await ml.testExecution.logTestStep('continue to the pick fields step'); + await ml.jobWizardCommon.assertConfigureDatafeedSectionExists(); + await ml.jobWizardCommon.advanceToPickFieldsSection(); + + await ml.testExecution.logTestStep('add detectors'); + await ml.jobWizardAdvanced.openCreateDetectorModal(); + await ml.jobWizardAdvanced.selectDetectorFunction('lat_long'); + await ml.jobWizardAdvanced.selectDetectorField('geo.coordinates'); + await ml.jobWizardAdvanced.setDetectorDescription('lat_long("geo.coordinates")'); + await ml.jobWizardAdvanced.confirmAddDetectorModal(); + + await ml.jobWizardAdvanced.openCreateDetectorModal(); + await ml.jobWizardAdvanced.selectDetectorFunction('high_sum'); + await ml.jobWizardAdvanced.selectDetectorField('bytes'); + await ml.jobWizardAdvanced.setDetectorDescription('sum(bytes)'); + await ml.jobWizardAdvanced.confirmAddDetectorModal(); + + await ml.testExecution.logTestStep('set the bucket span'); + await ml.jobWizardCommon.assertBucketSpanInputExists(); + await ml.jobWizardCommon.setBucketSpan('15m'); + + await ml.testExecution.logTestStep('set influencers'); + await ml.jobWizardCommon.assertInfluencerInputExists(); + await ml.jobWizardCommon.assertInfluencerSelection([]); + for (const influencer of ['geo.src', 'geo.dest', 'extension.keyword']) { + await ml.jobWizardCommon.addInfluencer(influencer); + } + + await ml.testExecution.logTestStep('set the model memory limit'); + await ml.jobWizardCommon.assertModelMemoryLimitInputExists({ + withAdvancedSection: false, + }); + await ml.jobWizardCommon.setModelMemoryLimit('11MB', { + withAdvancedSection: false, + }); + + await ml.testExecution.logTestStep('take screenshot'); + await mlScreenshots.removeFocusFromElement(); + await mlScreenshots.takeScreenshot('weblogs-advanced-wizard-geopoint', screenshotDirectories); + }); + + // the job stopped to produce an anomaly, needs investigation + it.skip('ecommerce anomaly explorer screenshots', async () => { + await ml.testExecution.logTestStep('navigate to job list'); + await ml.navigation.navigateToMl(); + await ml.navigation.navigateToJobManagement(); + await elasticChart.setNewChartUiDebugFlag(true); + + await ml.testExecution.logTestStep('open job in anomaly explorer'); + await ml.jobTable.waitForJobsToLoad(); + await ml.jobTable.filterWithSearchString(ecommerceGeoJobConfig.job_id, 1); + await ml.jobTable.clickOpenJobInAnomalyExplorerButton(ecommerceGeoJobConfig.job_id); + await ml.commonUI.waitForMlLoadingIndicatorToDisappear(); + + await ml.testExecution.logTestStep('select swim lane tile'); + const cells = await ml.swimLane.getCells(overallSwimLaneTestSubj); + const sampleCell = cells[0]; + await ml.swimLane.selectSingleCell(overallSwimLaneTestSubj, { + x: sampleCell.x + cellSize, + y: sampleCell.y + cellSize, + }); + await ml.swimLane.waitForSwimLanesToLoad(); + + await ml.testExecution.logTestStep('take screenshot'); + await ml.anomaliesTable.ensureDetailsOpen(0); + await ml.anomalyExplorer.scrollChartsContainerIntoView(); + + await mlScreenshots.takeScreenshot( + 'ecommerce-anomaly-explorer-geopoint', + screenshotDirectories + ); + }); + + it('weblogs anomaly explorer screenshots', async () => { + await ml.testExecution.logTestStep('navigate to job list'); + await ml.navigation.navigateToMl(); + await ml.navigation.navigateToJobManagement(); + await elasticChart.setNewChartUiDebugFlag(true); + + await ml.testExecution.logTestStep('open job in anomaly explorer'); + await ml.jobTable.waitForJobsToLoad(); + await ml.jobTable.filterWithSearchString(weblogGeoJobConfig.job_id, 1); + await ml.jobTable.clickOpenJobInAnomalyExplorerButton(weblogGeoJobConfig.job_id); + await ml.commonUI.waitForMlLoadingIndicatorToDisappear(); + + await ml.testExecution.logTestStep('select swim lane tile'); + const cells = await ml.swimLane.getCells(overallSwimLaneTestSubj); + const sampleCell1 = cells[11]; + const sampleCell2 = cells[12]; + await ml.swimLane.selectCells(overallSwimLaneTestSubj, { + x1: sampleCell1.x + cellSize, + y1: sampleCell1.y + cellSize, + x2: sampleCell2!.x + cellSize, + y2: sampleCell2!.y + cellSize, + }); + await ml.swimLane.waitForSwimLanesToLoad(); + + await ml.testExecution.logTestStep('set map options and take screenshot'); + await ml.anomalyExplorer.scrollChartsContainerIntoView(); + + // clickFitToData only works with displayed legend + await maps.openLegend(); + await maps.clickFitToData(); + await maps.closeLegend(); + + await mlScreenshots.takeScreenshot( + 'weblogs-anomaly-explorer-geopoint', + screenshotDirectories + ); + }); + }); +} diff --git a/x-pack/test/screenshot_creation/apps/ml_docs/anomaly_detection/index.ts b/x-pack/test/screenshot_creation/apps/ml_docs/anomaly_detection/index.ts new file mode 100644 index 0000000000000..389a240eaa464 --- /dev/null +++ b/x-pack/test/screenshot_creation/apps/ml_docs/anomaly_detection/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ loadTestFile }: FtrProviderContext) { + describe('anomaly detection', function () { + loadTestFile(require.resolve('./geographic_data')); + loadTestFile(require.resolve('./population_analysis')); + loadTestFile(require.resolve('./custom_urls')); + loadTestFile(require.resolve('./mapping_anomalies')); + }); +} diff --git a/x-pack/test/screenshot_creation/apps/ml_docs/anomaly_detection/mapping_anomalies.ts b/x-pack/test/screenshot_creation/apps/ml_docs/anomaly_detection/mapping_anomalies.ts new file mode 100644 index 0000000000000..6d653e0aed43e --- /dev/null +++ b/x-pack/test/screenshot_creation/apps/ml_docs/anomaly_detection/mapping_anomalies.ts @@ -0,0 +1,129 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../../ftr_provider_context'; +import { Job, Datafeed } from '../../../../../plugins/ml/common/types/anomaly_detection_jobs'; + +import { LOGS_INDEX_PATTERN } from '../index'; + +export default function ({ getPageObject, getService }: FtrProviderContext) { + const header = getPageObject('header'); + const maps = getPageObject('maps'); + const ml = getService('ml'); + const mlScreenshots = getService('mlScreenshots'); + const renderable = getService('renderable'); + + const screenshotDirectories = ['ml_docs', 'anomaly_detection']; + + const weblogVectorJobConfig = { + job_id: `weblogs-vectors`, + analysis_config: { + bucket_span: '15m', + influencers: ['geo.src', 'agent.keyword', 'geo.dest'], + detectors: [ + { + detector_description: 'Sum of bytes', + function: 'sum', + field_name: 'bytes', + partition_field_name: 'geo.dest', + }, + ], + }, + data_description: { time_field: 'timestamp', time_format: 'epoch_ms' }, + custom_settings: { created_by: 'multi-metric-wizard' }, + }; + + const weblogVectorDatafeedConfig = { + datafeed_id: 'datafeed-weblogs-vectors', + indices: [LOGS_INDEX_PATTERN], + job_id: 'weblogs-vectors', + query: { bool: { must: [{ match_all: {} }] } }, + }; + + describe('mapping anomalies', function () { + before(async () => { + await ml.api.createAndRunAnomalyDetectionLookbackJob( + weblogVectorJobConfig as Job, + weblogVectorDatafeedConfig as Datafeed + ); + }); + + after(async () => { + await ml.api.deleteAnomalyDetectionJobES(weblogVectorJobConfig.job_id); + await ml.api.cleanMlIndices(); + }); + + it('data visualizer screenshot', async () => { + await ml.testExecution.logTestStep('open index in data visualizer'); + await ml.navigation.navigateToMl(); + await ml.navigation.navigateToDataVisualizer(); + await ml.dataVisualizer.navigateToIndexPatternSelection(); + await ml.jobSourceSelection.selectSourceForIndexBasedDataVisualizer(LOGS_INDEX_PATTERN); + + await ml.testExecution.logTestStep('set data visualizer options'); + await ml.dataVisualizerIndexBased.assertTimeRangeSelectorSectionExists(); + await ml.dataVisualizerIndexBased.clickUseFullDataButton('14,074'); + await ml.dataVisualizerTable.setSampleSizeInputValue( + 'all', + 'geo.coordinates', + '14074 (100%)' + ); + await ml.dataVisualizerTable.setFieldNameFilter(['geo.dest']); + + await ml.testExecution.logTestStep('set maps options and take screenshot'); + await ml.dataVisualizerTable.ensureDetailsOpen('geo.dest'); + await renderable.waitForRender(); + await maps.openLegend(); + + await mlScreenshots.takeScreenshot( + 'weblogs-data-visualizer-choropleth', + screenshotDirectories + ); + }); + + it('wizard screenshot', async () => { + await ml.testExecution.logTestStep('navigate to job list'); + await ml.navigation.navigateToMl(); + await ml.navigation.navigateToJobManagement(); + + await ml.testExecution.logTestStep('load job in the multi-metric wizard'); + await ml.navigation.navigateToJobManagement(); + await ml.jobTable.filterWithSearchString(weblogVectorJobConfig.job_id, 1); + await ml.jobTable.clickCloneJobAction(weblogVectorJobConfig.job_id); + await ml.jobTypeSelection.assertMultiMetricJobWizardOpen(); + + await ml.testExecution.logTestStep('navigate to pick fields step'); + await ml.jobWizardCommon.advanceToPickFieldsSection(); + await header.awaitGlobalLoadingIndicatorHidden(); + await ml.jobWizardMultiMetric.scrollSplitFieldIntoView(); + + await ml.testExecution.logTestStep('take screenshot'); + await mlScreenshots.takeScreenshot( + 'weblogs-multimetric-wizard-vector', + screenshotDirectories + ); + }); + + it('anomaly explorer screenshot', async () => { + await ml.testExecution.logTestStep('navigate to job list'); + await ml.navigation.navigateToMl(); + await ml.navigation.navigateToJobManagement(); + + await ml.testExecution.logTestStep('open job in anomaly explorer'); + await ml.jobTable.waitForJobsToLoad(); + await ml.jobTable.filterWithSearchString(weblogVectorJobConfig.job_id, 1); + await ml.jobTable.clickOpenJobInAnomalyExplorerButton(weblogVectorJobConfig.job_id); + await ml.commonUI.waitForMlLoadingIndicatorToDisappear(); + + await ml.testExecution.logTestStep('scroll map into view and take screenshot'); + await ml.anomalyExplorer.scrollMapContainerIntoView(); + await renderable.waitForRender(); + await maps.openLegend(); + await mlScreenshots.takeScreenshot('weblogs-anomaly-explorer-vectors', screenshotDirectories); + }); + }); +} diff --git a/x-pack/test/screenshot_creation/apps/ml_docs/anomaly_detection/population_analysis.ts b/x-pack/test/screenshot_creation/apps/ml_docs/anomaly_detection/population_analysis.ts new file mode 100644 index 0000000000000..235b78ca8a662 --- /dev/null +++ b/x-pack/test/screenshot_creation/apps/ml_docs/anomaly_detection/population_analysis.ts @@ -0,0 +1,117 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../../ftr_provider_context'; +import { Job, Datafeed } from '../../../../../plugins/ml/common/types/anomaly_detection_jobs'; + +import { LOGS_INDEX_PATTERN } from '../index'; + +export default function ({ getService }: FtrProviderContext) { + const elasticChart = getService('elasticChart'); + const ml = getService('ml'); + const mlScreenshots = getService('mlScreenshots'); + const testSubjects = getService('testSubjects'); + + const screenshotDirectories = ['ml_docs', 'anomaly_detection']; + + const populationJobConfig = { + job_id: `population`, + analysis_config: { + bucket_span: '15m', + influencers: ['clientip'], + detectors: [ + { + function: 'mean', + field_name: 'bytes', + over_field_name: 'clientip', + }, + ], + }, + data_description: { time_field: 'timestamp', time_format: 'epoch_ms' }, + custom_settings: { created_by: 'population-wizard' }, + }; + + const populationDatafeedConfig = { + datafeed_id: 'datafeed-population', + indices: [LOGS_INDEX_PATTERN], + job_id: 'population', + query: { bool: { must: [{ match_all: {} }] } }, + }; + + const cellSize = 15; + const viewBySwimLaneTestSubj = 'mlAnomalyExplorerSwimlaneViewBy'; + + describe('population analysis', function () { + before(async () => { + await ml.api.createAndRunAnomalyDetectionLookbackJob( + populationJobConfig as Job, + populationDatafeedConfig as Datafeed + ); + }); + + after(async () => { + await elasticChart.setNewChartUiDebugFlag(false); + await ml.api.deleteAnomalyDetectionJobES(populationJobConfig.job_id); + await ml.api.cleanMlIndices(); + }); + + it('wizard screenshot', async () => { + await ml.testExecution.logTestStep('navigate to job list'); + await ml.navigation.navigateToMl(); + await ml.navigation.navigateToJobManagement(); + + await ml.testExecution.logTestStep('open job in wizard'); + await ml.jobTable.filterWithSearchString(populationJobConfig.job_id, 1); + await ml.jobTable.clickCloneJobAction(populationJobConfig.job_id); + await ml.jobTypeSelection.assertPopulationJobWizardOpen(); + + await ml.testExecution.logTestStep('continue to the pick fields step and take screenshot'); + await ml.jobWizardCommon.advanceToPickFieldsSection(); + await mlScreenshots.removeFocusFromElement(); + await mlScreenshots.takeScreenshot('ml-population-job', screenshotDirectories); + }); + + it('anomaly explorer screenshots', async () => { + await ml.testExecution.logTestStep('navigate to job list'); + await ml.navigation.navigateToMl(); + await ml.navigation.navigateToJobManagement(); + await elasticChart.setNewChartUiDebugFlag(true); + + await ml.testExecution.logTestStep('open job in anomaly explorer'); + await ml.jobTable.waitForJobsToLoad(); + await ml.jobTable.filterWithSearchString(populationJobConfig.job_id, 1); + await ml.jobTable.clickOpenJobInAnomalyExplorerButton(populationJobConfig.job_id); + await ml.commonUI.waitForMlLoadingIndicatorToDisappear(); + + await ml.testExecution.logTestStep('open tooltip and take screenshot'); + const viewBySwimLanes = await testSubjects.find(viewBySwimLaneTestSubj); + const cells = await ml.swimLane.getCells(viewBySwimLaneTestSubj); + const sampleCell = cells[0]; + + await viewBySwimLanes.moveMouseTo({ + xOffset: Math.floor(cellSize / 2.0), + yOffset: Math.floor(cellSize / 2.0), + }); + + await mlScreenshots.takeScreenshot('ml-population-results', screenshotDirectories); + + await ml.testExecution.logTestStep( + 'select swim lane tile, expand anomaly row and take screenshot' + ); + await ml.swimLane.selectSingleCell(viewBySwimLaneTestSubj, { + x: sampleCell.x + cellSize, + y: sampleCell.y + cellSize, + }); + await ml.swimLane.waitForSwimLanesToLoad(); + + await ml.anomalyExplorer.scrollChartsContainerIntoView(); + await ml.anomaliesTable.ensureDetailsOpen(0); + await ml.testExecution.logTestStep('take screenshot'); + await mlScreenshots.takeScreenshot('ml-population-anomaly', screenshotDirectories); + }); + }); +} diff --git a/x-pack/test/screenshot_creation/apps/ml_docs/data_frame_analytics/classification.ts b/x-pack/test/screenshot_creation/apps/ml_docs/data_frame_analytics/classification.ts new file mode 100644 index 0000000000000..ba78775a3a2d5 --- /dev/null +++ b/x-pack/test/screenshot_creation/apps/ml_docs/data_frame_analytics/classification.ts @@ -0,0 +1,163 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../../ftr_provider_context'; + +import { FLIGHTS_INDEX_PATTERN } from '../index'; +import { DataFrameAnalyticsConfig } from '../../../../../plugins/ml/public/application/data_frame_analytics/common'; +import { DeepPartial } from '../../../../../plugins/ml/common/types/common'; + +export default function ({ getService }: FtrProviderContext) { + const ml = getService('ml'); + const mlScreenshots = getService('mlScreenshots'); + + const screenshotDirectories = ['ml_docs', 'data_frame_analytics']; + + const classificationJobConfig: DeepPartial = { + id: 'model-flight-delays-classification', + source: { + index: FLIGHTS_INDEX_PATTERN, + }, + dest: { index: 'model-flight-delays-classification', results_field: 'ml' }, + analysis: { + classification: { + dependent_variable: 'FlightDelay', + training_percent: 10, + num_top_feature_importance_values: 10, + }, + }, + analyzed_fields: { + includes: [], + excludes: ['Cancelled', 'FlightDelayMin', 'FlightDelayType'], + }, + model_memory_limit: '1gb', + }; + + describe('classification job', function () { + before(async () => { + await ml.api.createAndRunDFAJob(classificationJobConfig as DataFrameAnalyticsConfig); + await ml.testResources.createIndexPatternIfNeeded(classificationJobConfig.dest!.index!); + }); + + after(async () => { + await ml.api.deleteDataFrameAnalyticsJobES(classificationJobConfig.id as string); + await ml.testResources.deleteIndexPatternByTitle(classificationJobConfig.dest!.index!); + await ml.api.deleteIndices(classificationJobConfig.dest!.index!); + await ml.api.cleanMlIndices(); + }); + + it('wizard screenshots', async () => { + await ml.testExecution.logTestStep('navigate to data frame analytics list'); + await ml.navigation.navigateToMl(); + await ml.navigation.navigateToDataFrameAnalytics(); + + await ml.testExecution.logTestStep('start new classification job creation in wizard'); + await ml.dataFrameAnalytics.startAnalyticsCreation(); + await ml.jobSourceSelection.selectSourceForAnalyticsJob(FLIGHTS_INDEX_PATTERN); + await ml.dataFrameAnalyticsCreation.assertConfigurationStepActive(); + + await ml.testExecution.logTestStep('select job type and set options'); + await ml.dataFrameAnalyticsCreation.assertJobTypeSelectExists(); + await ml.dataFrameAnalyticsCreation.selectJobType('classification'); + await ml.dataFrameAnalyticsCreation.assertDependentVariableInputExists(); + await ml.dataFrameAnalyticsCreation.selectDependentVariable('FlightDelay'); + + await ml.dataFrameAnalyticsCreation.assertSourceDataPreviewExists(); + await ml.dataFrameAnalyticsCreation.assertIncludeFieldsSelectionExists(); + + await ml.testExecution.logTestStep('take screenshot'); + await mlScreenshots.removeFocusFromElement(); + await ml.dataFrameAnalyticsCreation.scrollJobTypeSelectionIntoView(); + await mlScreenshots.takeScreenshot('flights-classification-job-1', screenshotDirectories); + + await ml.testExecution.logTestStep('scroll to scatterplot matrix and take screenshot'); + await ml.dataFrameAnalyticsCreation.assertScatterplotMatrixLoaded(); + await ml.dataFrameAnalyticsCreation.scrollScatterplotMatrixIntoView(); + await mlScreenshots.takeScreenshot( + 'flights-classification-scatterplot', + screenshotDirectories + ); + }); + + it('list row screenshot', async () => { + await ml.testExecution.logTestStep('navigate to data frame analytics list'); + await ml.navigation.navigateToMl(); + await ml.navigation.navigateToDataFrameAnalytics(); + + await ml.testExecution.logTestStep('open job row details'); + await ml.dataFrameAnalyticsTable.refreshAnalyticsTable(); + await ml.dataFrameAnalyticsTable.ensureDetailsOpen(classificationJobConfig.id as string); + await ml.dataFrameAnalyticsTable.ensureDetailsTabOpen( + classificationJobConfig.id as string, + 'job-details' + ); + + await ml.testExecution.logTestStep('take screenshot'); + await mlScreenshots.removeFocusFromElement(); + await mlScreenshots.takeScreenshot('flights-classification-details', screenshotDirectories); + }); + + it('results view screenshots', async () => { + await ml.testExecution.logTestStep('navigate to data frame analytics list'); + await ml.navigation.navigateToMl(); + await ml.navigation.navigateToDataFrameAnalytics(); + + await ml.testExecution.logTestStep('open job results view'); + await ml.dataFrameAnalyticsTable.refreshAnalyticsTable(); + await ml.dataFrameAnalyticsTable.filterWithSearchString( + classificationJobConfig.id as string, + 1 + ); + await ml.dataFrameAnalyticsTable.openResultsView(classificationJobConfig.id as string); + await ml.dataFrameAnalyticsResults.assertClassificationEvaluatePanelElementsExists(); + await ml.dataFrameAnalyticsResults.assertClassificationTablePanelExists(); + await ml.dataFrameAnalyticsResults.assertResultsTableExists(); + await ml.dataFrameAnalyticsResults.assertResultsTableNotEmpty(); + + await ml.testExecution.logTestStep('fold sections and take screenshot'); + await ml.dataFrameAnalyticsResults.expandAnalysisSection(false); + await ml.dataFrameAnalyticsResults.expandClassificationEvaluationSection(false); + await ml.dataFrameAnalyticsResults.expandFeatureImportanceSection(false); + await ml.dataFrameAnalyticsResults.expandScatterplotMatrixSection(false); + await ml.dataFrameAnalyticsResults.scrollAnalysisIntoView(); + await mlScreenshots.removeFocusFromElement(); + await mlScreenshots.takeScreenshot('flights-classification-results', screenshotDirectories); + + await ml.testExecution.logTestStep('expand feature importance section and take screenshot'); + await ml.dataFrameAnalyticsResults.expandFeatureImportanceSection(true); + await ml.dataFrameAnalyticsResults.scrollFeatureImportanceIntoView(); + await mlScreenshots.removeFocusFromElement(); + await mlScreenshots.takeScreenshot( + 'flights-classification-total-importance', + screenshotDirectories + ); + await ml.dataFrameAnalyticsResults.expandFeatureImportanceSection(false); + + await ml.testExecution.logTestStep('expand evaluation section and take screenshot'); + await ml.dataFrameAnalyticsResults.expandClassificationEvaluationSection(true); + await ml.dataFrameAnalyticsResults.scrollClassificationEvaluationIntoView(); + await mlScreenshots.removeFocusFromElement(); + await mlScreenshots.takeScreenshot( + 'flights-classification-evaluation', + screenshotDirectories + ); + await mlScreenshots.takeScreenshot('confusion-matrix-binary', screenshotDirectories); + await mlScreenshots.takeScreenshot('confusion-matrix-binary-accuracy', screenshotDirectories); + await ml.dataFrameAnalyticsResults.scrollRocCurveChartIntoView(); + await mlScreenshots.takeScreenshot('flights-classification-roc-curve', screenshotDirectories); + await ml.dataFrameAnalyticsResults.expandClassificationEvaluationSection(false); + + await ml.testExecution.logTestStep('open decision path popover and take screenshot'); + await ml.dataFrameAnalyticsResults.scrollResultsIntoView(); + await ml.dataFrameAnalyticsResults.openFeatureImportancePopover(); + await mlScreenshots.takeScreenshot( + 'flights-classification-importance', + screenshotDirectories + ); + }); + }); +} diff --git a/x-pack/test/screenshot_creation/apps/ml_docs/data_frame_analytics/index.ts b/x-pack/test/screenshot_creation/apps/ml_docs/data_frame_analytics/index.ts new file mode 100644 index 0000000000000..526d90c1c48bb --- /dev/null +++ b/x-pack/test/screenshot_creation/apps/ml_docs/data_frame_analytics/index.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ loadTestFile }: FtrProviderContext) { + describe('data frame analytics', function () { + loadTestFile(require.resolve('./outlier_detection')); + loadTestFile(require.resolve('./regression')); + loadTestFile(require.resolve('./classification')); + }); +} diff --git a/x-pack/test/screenshot_creation/apps/ml_docs/data_frame_analytics/outlier_detection.ts b/x-pack/test/screenshot_creation/apps/ml_docs/data_frame_analytics/outlier_detection.ts new file mode 100644 index 0000000000000..0571535432ef7 --- /dev/null +++ b/x-pack/test/screenshot_creation/apps/ml_docs/data_frame_analytics/outlier_detection.ts @@ -0,0 +1,132 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../../ftr_provider_context'; + +import { LOGS_INDEX_PATTERN } from '../index'; +import { DataFrameAnalyticsConfig } from '../../../../../plugins/ml/public/application/data_frame_analytics/common'; +import { DeepPartial } from '../../../../../plugins/ml/common/types/common'; + +export default function ({ getService }: FtrProviderContext) { + const ml = getService('ml'); + const mlScreenshots = getService('mlScreenshots'); + const transform = getService('transform'); + + const screenshotDirectories = ['ml_docs', 'data_frame_analytics']; + + const transformConfig = { + id: `logs-by-clientip`, + source: { index: LOGS_INDEX_PATTERN }, + pivot: { + group_by: { clientip: { terms: { field: 'clientip' } } }, + aggregations: { + '@timestamp.value_count': { value_count: { field: '@timestamp' } }, + 'bytes.max': { max: { field: 'bytes' } }, + 'bytes.sum': { sum: { field: 'bytes' } }, + 'request.value_count': { value_count: { field: 'request.keyword' } }, + }, + }, + description: 'Web logs by client IP', + dest: { index: 'weblog-clientip' }, + }; + + const outlierJobConfig: DeepPartial = { + id: 'weblog-outliers', + source: { index: 'weblog-clientip' }, + dest: { index: 'weblog-outliers', results_field: 'ml' }, + analysis: { outlier_detection: {} }, + analyzed_fields: { + includes: ['@timestamp.value_count', 'bytes.max', 'bytes.sum', 'request.value_count'], + excludes: [], + }, + model_memory_limit: '20mb', + }; + + describe('outlier detection job', function () { + before(async () => { + await transform.api.createAndRunTransform(transformConfig.id, transformConfig); + await ml.testResources.createIndexPatternIfNeeded(transformConfig.dest.index); + + await ml.api.createAndRunDFAJob(outlierJobConfig as DataFrameAnalyticsConfig); + await ml.testResources.createIndexPatternIfNeeded(outlierJobConfig.dest!.index!); + }); + + after(async () => { + await ml.testResources.deleteIndexPatternByTitle(transformConfig.dest.index); + await transform.api.deleteIndices(transformConfig.dest.index); + await transform.api.cleanTransformIndices(); + + await ml.api.deleteDataFrameAnalyticsJobES(outlierJobConfig.id as string); + await ml.testResources.deleteIndexPatternByTitle(outlierJobConfig.dest!.index!); + await ml.api.deleteIndices(outlierJobConfig.dest!.index!); + await ml.api.cleanMlIndices(); + }); + + it('transform screenshot', async () => { + await ml.testExecution.logTestStep('navigate to transform list'); + await transform.navigation.navigateTo(); + + await transform.testExecution.logTestStep('open transform in wizard'); + await transform.management.assertTransformListPageExists(); + await transform.table.refreshTransformList(); + await transform.table.filterWithSearchString(transformConfig.id, 1); + await transform.table.assertTransformRowActions(transformConfig.id, false); + await transform.table.clickTransformRowAction(transformConfig.id, 'Clone'); + await transform.wizard.assertDefineStepActive(); + + await ml.testExecution.logTestStep('take screenshot'); + await mlScreenshots.takeScreenshot('logs-transform-preview', screenshotDirectories); + }); + + it('wizard screenshots', async () => { + await ml.testExecution.logTestStep('navigate to data frame analytics list'); + await ml.navigation.navigateToMl(); + await ml.navigation.navigateToDataFrameAnalytics(); + + await ml.testExecution.logTestStep('open outlier detection job in wizard'); + await ml.dataFrameAnalyticsTable.waitForAnalyticsToLoad(); + await ml.dataFrameAnalyticsTable.filterWithSearchString(outlierJobConfig.id as string, 1); + await ml.dataFrameAnalyticsTable.cloneJob(outlierJobConfig.id as string); + await ml.dataFrameAnalyticsCreation.assertSourceDataPreviewExists(); + await ml.dataFrameAnalyticsCreation.assertIncludeFieldsSelectionExists(); + + await ml.testExecution.logTestStep('take screenshot'); + await mlScreenshots.takeScreenshot('weblog-outlier-job-1', screenshotDirectories); + + await ml.testExecution.logTestStep('scroll to scatterplot matrix and take screenshot'); + await ml.dataFrameAnalyticsCreation.assertScatterplotMatrixLoaded(); + await ml.dataFrameAnalyticsCreation.scrollScatterplotMatrixIntoView(); + await mlScreenshots.takeScreenshot('weblog-outlier-scatterplot', screenshotDirectories); + }); + + it('results view screenshots', async () => { + await ml.testExecution.logTestStep('navigate to data frame analytics list'); + await ml.navigation.navigateToMl(); + await ml.navigation.navigateToDataFrameAnalytics(); + + await ml.testExecution.logTestStep('open job results view'); + await ml.dataFrameAnalyticsTable.refreshAnalyticsTable(); + await ml.dataFrameAnalyticsTable.filterWithSearchString(outlierJobConfig.id as string, 1); + await ml.dataFrameAnalyticsTable.openResultsView(outlierJobConfig.id as string); + await ml.dataFrameAnalyticsResults.assertOutlierTablePanelExists(); + await ml.dataFrameAnalyticsResults.assertResultsTableExists(); + await ml.dataFrameAnalyticsResults.assertResultsTableNotEmpty(); + + await ml.testExecution.logTestStep('fold scatterplot section and take screenshot'); + await ml.dataFrameAnalyticsResults.expandScatterplotMatrixSection(false); + await mlScreenshots.removeFocusFromElement(); + await mlScreenshots.takeScreenshot('outliers', screenshotDirectories); + + await ml.testExecution.logTestStep('scroll to scatterplot matrix and take screenshot'); + await ml.dataFrameAnalyticsResults.expandScatterplotMatrixSection(true); + await mlScreenshots.removeFocusFromElement(); + await ml.dataFrameAnalyticsResults.assertScatterplotMatrixLoaded(); + await ml.dataFrameAnalyticsResults.scrollScatterplotMatrixIntoView(); + await mlScreenshots.takeScreenshot('outliers-scatterplot', screenshotDirectories); + }); + }); +} diff --git a/x-pack/test/screenshot_creation/apps/ml_docs/data_frame_analytics/regression.ts b/x-pack/test/screenshot_creation/apps/ml_docs/data_frame_analytics/regression.ts new file mode 100644 index 0000000000000..9b9f5f7a29b31 --- /dev/null +++ b/x-pack/test/screenshot_creation/apps/ml_docs/data_frame_analytics/regression.ts @@ -0,0 +1,152 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../../ftr_provider_context'; + +import { FLIGHTS_INDEX_PATTERN } from '../index'; +import { DataFrameAnalyticsConfig } from '../../../../../plugins/ml/public/application/data_frame_analytics/common'; +import { DeepPartial } from '../../../../../plugins/ml/common/types/common'; + +export default function ({ getService }: FtrProviderContext) { + const ml = getService('ml'); + const mlScreenshots = getService('mlScreenshots'); + + const screenshotDirectories = ['ml_docs', 'data_frame_analytics']; + + const regressionJobConfig: DeepPartial = { + id: 'model-flight-delays-regression', + source: { + index: FLIGHTS_INDEX_PATTERN, + query: { range: { DistanceKilometers: { gt: 0 } } }, + }, + dest: { index: 'model-flight-delays-regression', results_field: 'ml' }, + analysis: { + regression: { + dependent_variable: 'FlightDelayMin', + training_percent: 10, + num_top_feature_importance_values: 5, + }, + }, + analyzed_fields: { + includes: [], + excludes: ['Cancelled', 'FlightDelay', 'FlightDelayType'], + }, + model_memory_limit: '1gb', + }; + + describe('regression job', function () { + before(async () => { + await ml.api.createAndRunDFAJob(regressionJobConfig as DataFrameAnalyticsConfig); + await ml.testResources.createIndexPatternIfNeeded(regressionJobConfig.dest!.index!); + }); + + after(async () => { + await ml.api.deleteDataFrameAnalyticsJobES(regressionJobConfig.id as string); + await ml.testResources.deleteIndexPatternByTitle(regressionJobConfig.dest!.index!); + await ml.api.deleteIndices(regressionJobConfig.dest!.index!); + await ml.api.cleanMlIndices(); + }); + + it('wizard screenshots', async () => { + await ml.testExecution.logTestStep('navigate to data frame analytics list'); + await ml.navigation.navigateToMl(); + await ml.navigation.navigateToDataFrameAnalytics(); + + await ml.testExecution.logTestStep('start new regression job creation in wizard'); + await ml.dataFrameAnalytics.startAnalyticsCreation(); + await ml.jobSourceSelection.selectSourceForAnalyticsJob(FLIGHTS_INDEX_PATTERN); + await ml.dataFrameAnalyticsCreation.assertConfigurationStepActive(); + + await ml.testExecution.logTestStep('select job type and set options'); + await ml.dataFrameAnalyticsCreation.assertJobTypeSelectExists(); + await ml.dataFrameAnalyticsCreation.selectJobType('regression'); + await ml.dataFrameAnalyticsCreation.setQueryBarValue('DistanceKilometers > 0'); + await ml.dataFrameAnalyticsCreation.assertDependentVariableInputExists(); + await ml.dataFrameAnalyticsCreation.selectDependentVariable('FlightDelayMin'); + + await ml.dataFrameAnalyticsCreation.assertSourceDataPreviewExists(); + await ml.dataFrameAnalyticsCreation.assertIncludeFieldsSelectionExists(); + + await ml.testExecution.logTestStep('take screenshot'); + await mlScreenshots.removeFocusFromElement(); + await ml.dataFrameAnalyticsCreation.scrollJobTypeSelectionIntoView(); + await mlScreenshots.takeScreenshot('flights-regression-job-1', screenshotDirectories); + + await ml.testExecution.logTestStep('scroll to scatterplot matrix and take screenshot'); + await ml.dataFrameAnalyticsCreation.assertScatterplotMatrixLoaded(); + await ml.dataFrameAnalyticsCreation.scrollScatterplotMatrixIntoView(); + await mlScreenshots.takeScreenshot( + 'flightdata-regression-scatterplot', + screenshotDirectories + ); + }); + + it('list row screenshot', async () => { + await ml.testExecution.logTestStep('navigate to data frame analytics list'); + await ml.navigation.navigateToMl(); + await ml.navigation.navigateToDataFrameAnalytics(); + + await ml.testExecution.logTestStep('open job row details'); + await ml.dataFrameAnalyticsTable.refreshAnalyticsTable(); + await ml.dataFrameAnalyticsTable.ensureDetailsOpen(regressionJobConfig.id as string); + await ml.dataFrameAnalyticsTable.ensureDetailsTabOpen( + regressionJobConfig.id as string, + 'job-details' + ); + + await ml.testExecution.logTestStep('take screenshot'); + await mlScreenshots.removeFocusFromElement(); + await mlScreenshots.takeScreenshot('flights-regression-details', screenshotDirectories); + }); + + it('results view screenshots', async () => { + await ml.testExecution.logTestStep('navigate to data frame analytics list'); + await ml.navigation.navigateToMl(); + await ml.navigation.navigateToDataFrameAnalytics(); + + await ml.testExecution.logTestStep('open job results view'); + await ml.dataFrameAnalyticsTable.refreshAnalyticsTable(); + await ml.dataFrameAnalyticsTable.filterWithSearchString(regressionJobConfig.id as string, 1); + await ml.dataFrameAnalyticsTable.openResultsView(regressionJobConfig.id as string); + await ml.dataFrameAnalyticsResults.assertRegressionEvaluatePanelElementsExists(); + await ml.dataFrameAnalyticsResults.assertRegressionTablePanelExists(); + await ml.dataFrameAnalyticsResults.assertResultsTableExists(); + await ml.dataFrameAnalyticsResults.assertResultsTableNotEmpty(); + + await ml.testExecution.logTestStep('fold sections and take screenshot'); + await ml.dataFrameAnalyticsResults.expandAnalysisSection(false); + await ml.dataFrameAnalyticsResults.expandRegressionEvaluationSection(false); + await ml.dataFrameAnalyticsResults.expandFeatureImportanceSection(false); + await ml.dataFrameAnalyticsResults.expandScatterplotMatrixSection(false); + await ml.dataFrameAnalyticsResults.scrollAnalysisIntoView(); + await mlScreenshots.removeFocusFromElement(); + await mlScreenshots.takeScreenshot('flights-regression-results', screenshotDirectories); + + await ml.testExecution.logTestStep('expand feature importance section and take screenshot'); + await ml.dataFrameAnalyticsResults.expandFeatureImportanceSection(true); + await ml.dataFrameAnalyticsResults.scrollFeatureImportanceIntoView(); + await mlScreenshots.removeFocusFromElement(); + await mlScreenshots.takeScreenshot( + 'flights-regression-total-importance', + screenshotDirectories + ); + await ml.dataFrameAnalyticsResults.expandFeatureImportanceSection(false); + + await ml.testExecution.logTestStep('expand evaluation section and take screenshot'); + await ml.dataFrameAnalyticsResults.expandRegressionEvaluationSection(true); + await ml.dataFrameAnalyticsResults.scrollRegressionEvaluationIntoView(); + await mlScreenshots.removeFocusFromElement(); + await mlScreenshots.takeScreenshot('flights-regression-evaluation', screenshotDirectories); + await ml.dataFrameAnalyticsResults.expandRegressionEvaluationSection(false); + + await ml.testExecution.logTestStep('open decision path popover and take screenshot'); + await ml.dataFrameAnalyticsResults.scrollResultsIntoView(); + await ml.dataFrameAnalyticsResults.openFeatureImportancePopover(); + await mlScreenshots.takeScreenshot('flights-regression-importance', screenshotDirectories); + }); + }); +} diff --git a/x-pack/test/screenshot_creation/apps/ml_docs/index.ts b/x-pack/test/screenshot_creation/apps/ml_docs/index.ts new file mode 100644 index 0000000000000..9a12153682618 --- /dev/null +++ b/x-pack/test/screenshot_creation/apps/ml_docs/index.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../ftr_provider_context'; + +export const ECOMMERCE_INDEX_PATTERN = 'kibana_sample_data_ecommerce'; +export const FLIGHTS_INDEX_PATTERN = 'kibana_sample_data_flights'; +export const LOGS_INDEX_PATTERN = 'kibana_sample_data_logs'; + +export default function ({ getService, loadTestFile }: FtrProviderContext) { + const browser = getService('browser'); + const ml = getService('ml'); + + describe('machine learning docs', function () { + this.tags(['mlqa']); + + before(async () => { + await ml.testResources.installAllKibanaSampleData(); + await ml.testResources.setKibanaTimeZoneToUTC(); + await browser.setWindowSize(1920, 1080); + }); + + after(async () => { + await ml.testResources.removeAllKibanaSampleData(); + await ml.testResources.resetKibanaTimeZone(); + }); + + loadTestFile(require.resolve('./anomaly_detection')); + loadTestFile(require.resolve('./data_frame_analytics')); + }); +} diff --git a/x-pack/test/screenshot_creation/config.ts b/x-pack/test/screenshot_creation/config.ts new file mode 100644 index 0000000000000..659034e9fbe8b --- /dev/null +++ b/x-pack/test/screenshot_creation/config.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrConfigProviderContext } from '@kbn/test'; +import { services } from './services'; + +export default async function ({ readConfigFile }: FtrConfigProviderContext) { + const xpackFunctionalConfig = await readConfigFile(require.resolve('../functional/config.js')); + + return { + // default to the xpack functional config + ...xpackFunctionalConfig.getAll(), + services, + testFiles: [require.resolve('./apps')], + junit: { + ...xpackFunctionalConfig.get('junit'), + reportName: 'Chrome X-Pack UI Screenshot Creation', + }, + }; +} diff --git a/x-pack/test/screenshot_creation/ftr_provider_context.d.ts b/x-pack/test/screenshot_creation/ftr_provider_context.d.ts new file mode 100644 index 0000000000000..2cd67b6698a70 --- /dev/null +++ b/x-pack/test/screenshot_creation/ftr_provider_context.d.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { GenericFtrProviderContext } from '@kbn/test'; + +import { pageObjects } from '../functional/page_objects'; +import { services } from './services'; + +export type FtrProviderContext = GenericFtrProviderContext; diff --git a/x-pack/test/screenshot_creation/services/index.ts b/x-pack/test/screenshot_creation/services/index.ts new file mode 100644 index 0000000000000..dc5a107414415 --- /dev/null +++ b/x-pack/test/screenshot_creation/services/index.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { services as kibanaFunctionalServices } from '../../functional/services'; + +import { MachineLearningScreenshotsProvider } from './ml_screenshots'; + +export const services = { + ...kibanaFunctionalServices, + + mlScreenshots: MachineLearningScreenshotsProvider, +}; diff --git a/x-pack/test/screenshot_creation/services/ml_screenshots.ts b/x-pack/test/screenshot_creation/services/ml_screenshots.ts new file mode 100644 index 0000000000000..c0228c6a2b8b1 --- /dev/null +++ b/x-pack/test/screenshot_creation/services/ml_screenshots.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../ftr_provider_context'; + +export function MachineLearningScreenshotsProvider({ getService }: FtrProviderContext) { + const ml = getService('ml'); + const screenshot = getService('screenshots'); + + return { + async takeScreenshot(name: string, subDirectories: string[]) { + await screenshot.take(`${name}_new`, undefined, subDirectories); + }, + + async removeFocusFromElement() { + // open and close the Kibana nav to un-focus the last used element + await ml.navigation.openKibanaNav(); + await ml.navigation.closeKibanaNav(); + }, + }; +}