diff --git a/common/constants/explorer.ts b/common/constants/explorer.ts index d6fffd019..c0942c0a1 100644 --- a/common/constants/explorer.ts +++ b/common/constants/explorer.ts @@ -93,6 +93,7 @@ export const REDUX_EXPL_SLICE_QUERY_TABS = 'queryTabs'; export const REDUX_EXPL_SLICE_VISUALIZATION = 'explorerVisualization'; export const REDUX_EXPL_SLICE_COUNT_DISTRIBUTION = 'countDistributionVisualization'; export const REDUX_EXPL_SLICE_PATTERNS = 'patterns'; +export const REDUX_EXPL_SLICE_SEARCH_META_DATA = 'searchMetaData'; export const PLOTLY_GAUGE_COLUMN_NUMBER = 4; export const APP_ANALYTICS_TAB_ID_REGEX = /application-analytics-tab.+/; export const DEFAULT_AVAILABILITY_QUERY = 'stats count() by span( timestamp, 1h )'; diff --git a/common/constants/shared.ts b/common/constants/shared.ts index 2b36060c8..ddcffeb30 100644 --- a/common/constants/shared.ts +++ b/common/constants/shared.ts @@ -238,3 +238,5 @@ export const VISUALIZATION_ERROR = { NO_DATA: 'No data found.', INVALID_DATA: 'Invalid visualization data', }; + +export const S3_DATASOURCE_TYPE = 'S3_DATASOURCE'; diff --git a/public/components/app.tsx b/public/components/app.tsx index 805ebc2e0..4569c4261 100644 --- a/public/components/app.tsx +++ b/public/components/app.tsx @@ -57,6 +57,7 @@ export const App = ({ timestampUtils, queryManager, startPage, + dataSourcePluggables, }: ObservabilityAppDeps) => { const { chrome, http, notifications, savedObjects: coreSavedObjects } = CoreStartProp; const parentBreadcrumb = { @@ -87,6 +88,7 @@ export const App = ({ parentBreadcrumb={parentBreadcrumb} parentBreadcrumbs={[parentBreadcrumb]} setBreadcrumbs={chrome.setBreadcrumbs} + dataSourcePluggables={dataSourcePluggables} /> diff --git a/public/components/common/search/autocomplete.tsx b/public/components/common/search/autocomplete.tsx index a8c4007a7..4b9ab22cb 100644 --- a/public/components/common/search/autocomplete.tsx +++ b/public/components/common/search/autocomplete.tsx @@ -28,6 +28,7 @@ interface AutocompleteProps extends IQueryBarProps { placeholder?: string; possibleCommands?: Array<{ label: string }>; append?: any; + isSuggestionDisabled?: boolean; } export const Autocomplete = (props: AutocompleteProps) => { @@ -45,6 +46,7 @@ export const Autocomplete = (props: AutocompleteProps) => { placeholder = 'Enter PPL query', possibleCommands, append, + isSuggestionDisabled = false, } = props; const [autocompleteState, setAutocompleteState] = useState>({ @@ -143,7 +145,7 @@ export const Autocomplete = (props: AutocompleteProps) => { {...(panelsFilter && { append, fullWidth: true })} disabled={isDisabled} /> - {autocompleteState.isOpen && ( + {autocompleteState.isOpen && !isSuggestionDisabled && (
{ +describe.skip('Search bar', () => { it('handles query change', () => { const query = 'rawQuery'; const tempQuery = 'rawQuery'; @@ -62,7 +62,7 @@ describe('Search bar', () => { popoverItems={popoverItems} isLiveTailOn={isLiveTailOn} countDistribution={countDistribution} - curVisId={'line'} + curVisId={'line'} spanValue={false} setSubType={'metric'} setMetricMeasure={'hours (h)'} diff --git a/public/components/common/search/search.tsx b/public/components/common/search/search.tsx index e22f571b4..359e38689 100644 --- a/public/components/common/search/search.tsx +++ b/public/components/common/search/search.tsx @@ -5,8 +5,9 @@ import './search.scss'; -import React, { useState } from 'react'; -import { isEqual } from 'lodash'; +import React, { useState, useEffect } from 'react'; +import { useDispatch, useSelector } from 'react-redux'; +import { isEqual, lowerCase } from 'lodash'; import { EuiFlexGroup, EuiButton, @@ -24,9 +25,18 @@ import { Autocomplete } from './autocomplete'; import { SavePanel } from '../../event_analytics/explorer/save_panel'; import { PPLReferenceFlyout } from '../helpers'; import { uiSettingsService } from '../../../../common/utils'; -import { APP_ANALYTICS_TAB_ID_REGEX } from '../../../../common/constants/explorer'; +import { APP_ANALYTICS_TAB_ID_REGEX, RAW_QUERY } from '../../../../common/constants/explorer'; import { LiveTailButton, StopLiveButton } from '../live_tail/live_tail_button'; import { PPL_SPAN_REGEX } from '../../../../common/constants/shared'; +import { coreRefs } from '../../../framework/core_refs'; +import { useFetchEvents } from '../../../components/event_analytics/hooks'; +import { SQLService } from '../../../services/requests/sql'; +import { + selectSearchMetaData, + update as updateSearchMetaData, +} from '../../event_analytics/redux/slices/search_meta_data_slice'; +import { usePolling } from '../../../components/hooks/use_polling'; +import { changeQuery } from '../../../components/event_analytics/redux/slices/query_slice'; export interface IQueryBarProps { query: string; tempQuery: string; @@ -51,7 +61,6 @@ export const Search = (props: any) => { query, tempQuery, handleQueryChange, - handleQuerySearch, handleTimePickerChange, dslService, startTime, @@ -84,11 +93,34 @@ export const Search = (props: any) => { liveTailName, curVisId, setSubType, + setIsQueryRunning, } = props; + const explorerSearchMetadata = useSelector(selectSearchMetaData)[tabId]; + const dispatch = useDispatch(); const appLogEvents = tabId.match(APP_ANALYTICS_TAB_ID_REGEX); const [isSavePanelOpen, setIsSavePanelOpen] = useState(false); const [isFlyoutVisible, setIsFlyoutVisible] = useState(false); + const [queryLang, setQueryLang] = useState([]); + const [jobId, setJobId] = useState(''); + const sqlService = new SQLService(coreRefs.http); + const { application } = coreRefs; + + const { + data: pollingResult, + loading: pollingLoading, + error: pollingError, + startPolling, + stopPolling, + } = usePolling((params) => { + return sqlService.fetchWithJobId(params); + }, 5000); + + const requestParams = { tabId }; + const { getLiveTail, getEvents, getAvailableFields, dispatchOnGettingHis } = useFetchEvents({ + pplService: new SQLService(coreRefs.http), + requestParams, + }); const closeFlyout = () => { setIsFlyoutVisible(false); @@ -128,6 +160,50 @@ export const Search = (props: any) => { /> ); + const handleQueryLanguageChange = (lang) => { + if (lang[0].label === 'DQL') { + return application.navigateToUrl( + `../app/data-explorer/discover#?_a=(discover:(columns:!(_source),isDirty:!f,sort:!()),metadata:(indexPattern:'${explorerSearchMetadata.datasources[0].value}',view:discover))&_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now-15m,to:now))&_q=(filters:!(),query:(language:kuery,query:''))` + ); + } + dispatch( + updateSearchMetaData({ + tabId, + data: { lang: lang[0].label }, + }) + ); + setQueryLang(lang); + }; + + const onQuerySearch = (lang) => { + handleTimeRangePickerRefresh(); + }; + + useEffect(() => { + if (pollingResult && (pollingResult.status === 'SUCCESS' || pollingResult.datarows)) { + // update page with data + dispatchOnGettingHis(pollingResult, ''); + // stop polling + stopPolling(); + setIsQueryRunning(false); + } + }, [pollingResult, pollingError]); + + useEffect(() => { + if (explorerSearchMetadata.datasources?.[0]?.type === 'DEFAULT_INDEX_PATTERNS') { + const queryWithSelectedSource = `source = ${explorerSearchMetadata.datasources[0].label}`; + handleQueryChange(queryWithSelectedSource); + dispatch( + changeQuery({ + tabId, + query: { + [RAW_QUERY]: queryWithSelectedSource, + }, + }) + ); + } + }, [explorerSearchMetadata.datasources]); + return (
@@ -140,14 +216,25 @@ export const Search = (props: any) => { )} - + + + + { + onQuerySearch(queryLang); + }} dslService={dslService} getSuggestions={getSuggestions} onItemSelect={onItemSelect} @@ -177,7 +264,9 @@ export const Search = (props: any) => { liveStreamChecked={props.liveStreamChecked} onLiveStreamChange={props.onLiveStreamChange} handleTimePickerChange={(timeRange: string[]) => handleTimePickerChange(timeRange)} - handleTimeRangePickerRefresh={handleTimeRangePickerRefresh} + handleTimeRangePickerRefresh={() => { + onQuerySearch(queryLang); + }} /> )} diff --git a/public/components/common/search/sql_search.tsx b/public/components/common/search/sql_search.tsx new file mode 100644 index 000000000..14e57def3 --- /dev/null +++ b/public/components/common/search/sql_search.tsx @@ -0,0 +1,348 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import './search.scss'; + +import React, { useState, useEffect } from 'react'; +import { useDispatch, useSelector } from 'react-redux'; +import { isEqual, lowerCase } from 'lodash'; +import { + EuiFlexGroup, + EuiButton, + EuiFlexItem, + EuiPopover, + EuiButtonEmpty, + EuiPopoverFooter, + EuiBadge, + EuiToolTip, + EuiComboBox, + EuiTextArea, +} from '@elastic/eui'; +import { DatePicker } from './date_picker'; +import '@algolia/autocomplete-theme-classic'; +import { Autocomplete } from './autocomplete'; +import { SavePanel } from '../../event_analytics/explorer/save_panel'; +import { PPLReferenceFlyout } from '../helpers'; +import { uiSettingsService } from '../../../../common/utils'; +import { APP_ANALYTICS_TAB_ID_REGEX, RAW_QUERY } from '../../../../common/constants/explorer'; +import { PPL_SPAN_REGEX } from '../../../../common/constants/shared'; +import { coreRefs } from '../../../framework/core_refs'; +import { useFetchEvents } from '../../../components/event_analytics/hooks'; +import { SQLService } from '../../../services/requests/sql'; +import { usePolling } from '../../../components/hooks/use_polling'; +import { + selectSearchMetaData, + update as updateSearchMetaData, +} from '../../event_analytics/redux/slices/search_meta_data_slice'; +export interface IQueryBarProps { + query: string; + tempQuery: string; + handleQueryChange: (query: string) => void; + handleQuerySearch: () => void; + dslService: any; +} + +export interface IDatePickerProps { + startTime: string; + endTime: string; + setStartTime: () => void; + setEndTime: () => void; + setTimeRange: () => void; + setIsOutputStale: () => void; + handleTimePickerChange: (timeRange: string[]) => any; + handleTimeRangePickerRefresh: () => any; +} + +export const DirectSearch = (props: any) => { + const { + query, + tempQuery, + handleQueryChange, + handleTimePickerChange, + dslService, + startTime, + endTime, + setStartTime, + setEndTime, + setIsOutputStale, + selectedPanelName, + selectedCustomPanelOptions, + setSelectedPanelName, + setSelectedCustomPanelOptions, + handleSavingObject, + isPanelTextFieldInvalid, + savedObjects, + showSavePanelOptionsList, + showSaveButton = true, + handleTimeRangePickerRefresh, + isLiveTailPopoverOpen, + closeLiveTailPopover, + popoverItems, + isLiveTailOn, + selectedSubTabId, + searchBarConfigs = {}, + getSuggestions, + onItemSelect, + tabId = '', + baseQuery = '', + stopLive, + setIsLiveTailPopoverOpen, + liveTailName, + curVisId, + setSubType, + setIsQueryRunning, + } = props; + + const explorerSearchMetadata = useSelector(selectSearchMetaData)[tabId]; + const dispatch = useDispatch(); + const appLogEvents = tabId.match(APP_ANALYTICS_TAB_ID_REGEX); + const [isSavePanelOpen, setIsSavePanelOpen] = useState(false); + const [isFlyoutVisible, setIsFlyoutVisible] = useState(false); + const [queryLang, setQueryLang] = useState([]); + const [jobId, setJobId] = useState(''); + const sqlService = new SQLService(coreRefs.http); + const { application } = coreRefs; + + const { + data: pollingResult, + loading: pollingLoading, + error: pollingError, + startPolling, + stopPolling, + } = usePolling((params) => { + return sqlService.fetchWithJobId(params); + }, 5000); + + const requestParams = { tabId }; + const { getLiveTail, getEvents, getAvailableFields, dispatchOnGettingHis } = useFetchEvents({ + pplService: new SQLService(coreRefs.http), + requestParams, + }); + + const closeFlyout = () => { + setIsFlyoutVisible(false); + }; + + const showFlyout = () => { + setIsFlyoutVisible(true); + }; + + let flyout; + if (isFlyoutVisible) { + flyout = ; + } + + const Savebutton = ( + { + setIsSavePanelOpen((staleState) => { + return !staleState; + }); + }} + data-test-subj="eventExplorer__saveManagementPopover" + iconType="arrowDown" + > + Save + + ); + + const handleQueryLanguageChange = (lang) => { + if (lang[0].label === 'DQL') { + return application.navigateToUrl( + `../app/data-explorer/discover#?_a=(discover:(columns:!(_source),isDirty:!f,sort:!()),metadata:(indexPattern:'${explorerSearchMetadata.datasources[0].value}',view:discover))&_g=(filters:!(),refreshInterval:(pause:!t,value:0),time:(from:now-15m,to:now))&_q=(filters:!(),query:(language:kuery,query:''))` + ); + } + dispatch( + updateSearchMetaData({ + tabId, + data: { lang: lang[0].label }, + }) + ); + setQueryLang(lang); + }; + + const onQuerySearch = (lang) => { + setIsQueryRunning(true); + dispatch( + updateSearchMetaData({ + tabId, + data: { + isPolling: true, + }, + }) + ); + sqlService + .fetch({ + lang: lowerCase(lang[0].label), + query: tempQuery || query, + datasource: explorerSearchMetadata.datasources[0].name, + }) + .then((result) => { + if (result.queryId) { + setJobId(result.queryId); + startPolling({ + queryId: result.queryId, + }); + } else { + console.log('no query id found in response'); + } + }) + .catch((e) => { + setIsQueryRunning(false); + console.error(e); + }) + .finally(() => {}); + }; + + useEffect(() => { + // cancel direct query + if (pollingResult && (pollingResult.status === 'SUCCESS' || pollingResult.datarows)) { + // stop polling + stopPolling(); + setIsQueryRunning(false); + dispatch( + updateSearchMetaData({ + tabId, + data: { + isPolling: false, + }, + }) + ); + // update page with data + dispatchOnGettingHis(pollingResult, ''); + } + }, [pollingResult, pollingError]); + + useEffect(() => { + if (explorerSearchMetadata.isPolling === false) { + stopPolling(); + setIsQueryRunning(false); + } + }, [explorerSearchMetadata.isPolling]); + + return ( +
+ + {appLogEvents && ( + + + + Base Query + + + + )} + + + + + { + onQuerySearch(queryLang); + }} + dslService={dslService} + getSuggestions={getSuggestions} + onItemSelect={onItemSelect} + tabId={tabId} + isSuggestionDisabled={queryLang[0]?.label === 'SQL'} + /> + {queryLang[0]?.label && ( + showFlyout()} + onClickAriaLabel={'pplLinkShowFlyout'} + > + PPL + + )} + + + + { + onQuerySearch(queryLang); + }} + fill + > + Search + + + + {showSaveButton && searchBarConfigs[selectedSubTabId]?.showSaveButton && ( + <> + + setIsSavePanelOpen(false)} + > + + + + + setIsSavePanelOpen(false)} + data-test-subj="eventExplorer__querySaveCancel" + > + Cancel + + + + { + handleSavingObject(); + setIsSavePanelOpen(false); + }} + data-test-subj="eventExplorer__querySaveConfirm" + > + Save + + + + + + + + )} + + {flyout} +
+ ); +}; diff --git a/public/components/datasources/components/__tests__/testing_constants.ts b/public/components/datasources/components/__tests__/testing_constants.ts new file mode 100644 index 000000000..5cad33957 --- /dev/null +++ b/public/components/datasources/components/__tests__/testing_constants.ts @@ -0,0 +1,75 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export const showDatasourceData = [ + { + name: 'my_spark3', + connector: 'SPARK', + allowedRoles: [], + properties: { + 'spark.connector': 'emr', + 'spark.datasource.flint.host': '0.0.0.0', + 'spark.datasource.flint.integration': + 'https://aws.oss.sonatype.org/content/repositories/snapshots/org/opensearch/opensearch-spark-standalone_2.12/0.1.0-SNAPSHOT/opensearch-spark-standalone_2.12-0.1.0-20230731.182705-3.jar', + 'spark.datasource.flint.port': '9200', + 'spark.datasource.flint.scheme': 'http', + 'emr.cluster': 'j-3UNQLT1MPBGLG', + }, + }, + { + name: 'my_spark4', + connector: 'SPARK', + allowedRoles: [], + properties: { + 'spark.connector': 'emr', + 'spark.datasource.flint.host': '15.248.1.68', + 'spark.datasource.flint.integration': + 'https://aws.oss.sonatype.org/content/repositories/snapshots/org/opensearch/opensearch-spark-standalone_2.12/0.1.0-SNAPSHOT/opensearch-spark-standalone_2.12-0.1.0-20230731.182705-3.jar', + 'spark.datasource.flint.port': '9200', + 'spark.datasource.flint.scheme': 'http', + 'emr.cluster': 'j-3UNQLT1MPBGLG', + }, + }, + { + name: 'my_spark', + connector: 'SPARK', + allowedRoles: [], + properties: { + 'spark.connector': 'emr', + 'spark.datasource.flint.host': '0.0.0.0', + 'spark.datasource.flint.port': '9200', + 'spark.datasource.flint.scheme': 'http', + 'spark.datasource.flint.region': 'xxx', + 'emr.cluster': 'xxx', + }, + }, + { + name: 'my_spark2', + connector: 'SPARK', + allowedRoles: [], + properties: { + 'spark.connector': 'emr', + 'spark.datasource.flint.host': '0.0.0.0', + 'spark.datasource.flint.port': '9200', + 'spark.datasource.flint.scheme': 'http', + 'emr.cluster': 'j-3UNQLT1MPBGLG', + }, + }, +]; + +export const describeDatasource = { + name: 'my_spark3', + connector: 'SPARK', + allowedRoles: [], + properties: { + 'spark.connector': 'emr', + 'spark.datasource.flint.host': '0.0.0.0', + 'spark.datasource.flint.integration': + 'https://aws.oss.sonatype.org/content/repositories/snapshots/org/opensearch/opensearch-spark-standalone_2.12/0.1.0-SNAPSHOT/opensearch-spark-standalone_2.12-0.1.0-20230731.182705-3.jar', + 'spark.datasource.flint.port': '9200', + 'spark.datasource.flint.scheme': 'http', + 'emr.cluster': 'j-3UNQLT1MPBGLG', + }, +}; diff --git a/public/components/event_analytics/explorer/datasources/datasources_selection.tsx b/public/components/event_analytics/explorer/datasources/datasources_selection.tsx new file mode 100644 index 000000000..86b0fcf1a --- /dev/null +++ b/public/components/event_analytics/explorer/datasources/datasources_selection.tsx @@ -0,0 +1,129 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +import React, { useCallback, useEffect, useState, useContext } from 'react'; +import { batch, useDispatch, useSelector } from 'react-redux'; +import { DataSourceSelectable } from '../../../../../../../src/plugins/data/public'; +import { + selectSearchMetaData, + update as updateSearchMetaData, +} from '../../../event_analytics/redux/slices/search_meta_data_slice'; +import { coreRefs } from '../../../../framework/core_refs'; +import { reset as resetFields } from '../../redux/slices/field_slice'; +import { reset as resetPatterns } from '../../redux/slices/patterns_slice'; +import { reset as resetQueryResults } from '../../redux/slices/query_result_slice'; +import { reset as resetVisConfig } from '../../redux/slices/viualization_config_slice'; +import { reset as resetVisualization } from '../../redux/slices/visualization_slice'; +import { reset as resetCountDistribution } from '../../redux/slices/count_distribution_slice'; +import { LogExplorerRouterContext } from '../..'; + +export const DataSourceSelection = ({ tabId }) => { + const { dataSources } = coreRefs; + const dispatch = useDispatch(); + const routerContext = useContext(LogExplorerRouterContext); + const explorerSearchMetadata = useSelector(selectSearchMetaData)[tabId]; + const [activeDataSources, setActiveDataSources] = useState([]); + const [dataSourceOptionList, setDataSourceOptionList] = useState([]); + const [selectedSources, setSelectedSources] = useState([...explorerSearchMetadata.datasources]); + + const resetStateOnDatasourceChange = () => { + dispatch( + resetFields({ + tabId, + }) + ); + dispatch( + resetPatterns({ + tabId, + }) + ); + dispatch( + resetQueryResults({ + tabId, + }) + ); + dispatch( + resetVisConfig({ + tabId, + }) + ); + dispatch( + resetVisualization({ + tabId, + }) + ); + dispatch( + resetCountDistribution({ + tabId, + }) + ); + }; + + const handleSourceChange = (selectedSource) => { + batch(() => { + resetStateOnDatasourceChange(); + dispatch( + updateSearchMetaData({ + tabId, + data: { + datasources: selectedSource, + }, + }) + ); + }); + setSelectedSources(selectedSource); + }; + + useEffect(() => { + setSelectedSources([...(explorerSearchMetadata.datasources || [])]); + return () => {}; + }, [explorerSearchMetadata.datasources]); + + const handleDataSetFetchError = useCallback(() => { + return (error) => {}; + }, []); + + useEffect(() => { + const subscription = dataSources.dataSourceService.dataSources$.subscribe( + (currentDataSources) => { + setActiveDataSources([...Object.values(currentDataSources)]); + } + ); + + return () => subscription.unsubscribe(); + }, []); + + useEffect(() => { + // update datasource if url contains + const datasourceName = routerContext?.searchParams.get('datasourceName'); + const datasourceType = routerContext?.searchParams.get('datasourceType'); + if (datasourceName && datasourceType) { + dispatch( + updateSearchMetaData({ + tabId, + data: { + datasources: [ + { + label: datasourceName, + type: datasourceType, + }, + ], + }, + }) + ); + } + }, []); + + return ( + + ); +}; diff --git a/public/components/event_analytics/explorer/direct_query_running.tsx b/public/components/event_analytics/explorer/direct_query_running.tsx new file mode 100644 index 000000000..9536909ce --- /dev/null +++ b/public/components/event_analytics/explorer/direct_query_running.tsx @@ -0,0 +1,36 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React from 'react'; +import { useDispatch } from 'react-redux'; +import { EuiProgress, EuiEmptyPrompt, EuiButton } from '@elastic/eui'; +import { update as updateSearchMetaData } from '../redux/slices/search_meta_data_slice'; + +export const DirectQueryRunning = ({ tabId }: { tabId: string }) => { + const dispatch = useDispatch(); + return ( + } + title={

Query Processing

} + body={ + { + dispatch( + updateSearchMetaData({ + tabId, + data: { + isPolling: false, + }, + }) + ); + }} + > + Cancel + + } + /> + ); +}; diff --git a/public/components/event_analytics/explorer/explorer.scss b/public/components/event_analytics/explorer/explorer.scss index 9f2d60198..8617e26df 100644 --- a/public/components/event_analytics/explorer/explorer.scss +++ b/public/components/event_analytics/explorer/explorer.scss @@ -17,4 +17,13 @@ .mainContentTabs .euiResizableContainer { height: calc(100vh - 298px); } - \ No newline at end of file + +.explorer-loading-spinner { + position: relative; + left: 50%; + top: 50vh; + width: 20px; + height: 20px; + margin-left: -5vw; + margin-top: -20vh; +} diff --git a/public/components/event_analytics/explorer/explorer.tsx b/public/components/event_analytics/explorer/explorer.tsx index d0a3075cb..aae1a2de4 100644 --- a/public/components/event_analytics/explorer/explorer.tsx +++ b/public/components/event_analytics/explorer/explorer.tsx @@ -5,11 +5,9 @@ import dateMath from '@elastic/datemath'; import { - EuiButtonIcon, EuiContextMenuItem, EuiFlexGroup, EuiFlexItem, - EuiHorizontalRule, EuiLink, EuiLoadingSpinner, EuiPage, @@ -20,7 +18,6 @@ import { EuiTabbedContent, EuiTabbedContentTab, EuiText, - EuiTitle, } from '@elastic/eui'; import { FormattedMessage } from '@osd/i18n/react'; import classNames from 'classnames'; @@ -113,6 +110,10 @@ import { change as updateVizConfig, selectVisualizationConfig, } from '../redux/slices/viualization_config_slice'; +import { + update as updateSearchMetaData, + selectSearchMetaData, +} from '../../event_analytics/redux/slices/search_meta_data_slice'; import { formatError, getDefaultVisConfig } from '../utils'; import { getContentTabTitle, getDateRange } from '../utils/utils'; import { DataGrid } from './events_views/data_grid'; @@ -123,6 +124,9 @@ import { Sidebar } from './sidebar'; import { TimechartHeader } from './timechart_header'; import { ExplorerVisualizations } from './visualizations'; import { CountDistribution } from './visualizations/count_distribution'; +import { DataSourceSelection } from './datasources/datasources_selection'; +import { DirectQueryRunning } from './direct_query_running'; +import { DirectQueryVisualization } from './visualizations/direct_query_vis'; export const Explorer = ({ pplService, @@ -147,6 +151,7 @@ export const Explorer = ({ callback, callbackInApp, queryManager = new QueryManager(), + dataSourcePluggables, }: IExplorerProps) => { const routerContext = useContext(LogExplorerRouterContext); const dispatch = useDispatch(); @@ -167,6 +172,7 @@ export const Explorer = ({ pplService, requestParams, }); + const appLogEvents = tabId.startsWith('application-analytics-tab'); const query = useSelector(selectQueries)[tabId]; const explorerData = useSelector(selectQueryResult)[tabId]; @@ -174,6 +180,7 @@ export const Explorer = ({ const countDistribution = useSelector(selectCountDistribution)[tabId]; const explorerVisualizations = useSelector(selectExplorerVisualization)[tabId]; const userVizConfigs = useSelector(selectVisualizationConfig)[tabId] || {}; + const explorerSearchMeta = useSelector(selectSearchMetaData)[tabId]; const [selectedContentTabId, setSelectedContentTab] = useState(TAB_EVENT_ID); const [selectedCustomPanelOptions, setSelectedCustomPanelOptions] = useState([]); const [selectedPanelName, setSelectedPanelName] = useState(''); @@ -192,6 +199,17 @@ export const Explorer = ({ const [browserTabFocus, setBrowserTabFocus] = useState(true); const [liveTimestamp, setLiveTimestamp] = useState(DATE_PICKER_FORMAT); const [triggerAvailability, setTriggerAvailability] = useState(false); + const [isQueryRunning, setIsQueryRunning] = useState(false); + const currentPluggable = useMemo(() => { + return ( + dataSourcePluggables[explorerSearchMeta.datasources[0]?.type] || + dataSourcePluggables.DEFAULT_INDEX_PATTERNS + ); + }, [explorerSearchMeta.datasources]); + const { ui } = + currentPluggable?.getComponentSetForVariation('languages', explorerSearchMeta.lang || 'SQL') || + {}; + const SearchBar = ui?.SearchBar || Search; const selectedIntervalRef = useRef<{ text: string; @@ -412,10 +430,6 @@ export const Explorer = ({ } }; - const sidebarClassName = classNames({ - closed: isSidebarClosed, - }); - const mainSectionClassName = classNames({ 'col-md-8': !isSidebarClosed, 'col-md-12': isSidebarClosed, @@ -470,178 +484,117 @@ export const Explorer = ({ const mainContent = useMemo(() => { return ( - <> - - - {!isSidebarClosed && ( -
- 0 - ? storedExplorerFields - : explorerFields - } - setStoredExplorerFields={setStoredExplorerFields} - /> -
- )} - { - setIsSidebarClosed((staleState) => { - return !staleState; - }); - }} - data-test-subj="collapseSideBarButton" - aria-controls="discover-sidebar" - aria-expanded={isSidebarClosed ? 'false' : 'true'} - aria-label="Toggle sidebar" - className="dscCollapsibleSidebar__collapseButton" - /> - - - {explorerData && !isEmpty(explorerData.jsonData) ? ( - - - - - {countDistribution?.data && !isLiveTailOnRef.current && ( - <> - {}} - /> - { - const intervalOptionsIndex = timeIntervalOptions.findIndex( - (item) => item.value === selectedIntrv - ); - const intrv = selectedIntrv.replace(/^auto_/, ''); - getCountVisualizations(intrv); - selectedIntervalRef.current = - timeIntervalOptions[intervalOptionsIndex]; - getPatterns(intrv, getErrorHandler('Error fetching patterns')); - }} - stateInterval={selectedIntervalRef.current?.value} - startTime={appLogEvents ? startTime : dateRange[0]} - endTime={appLogEvents ? endTime : dateRange[1]} - /> - - - )} - - - - - - - - - - - - - - - -
-

- -

-
- {isLiveTailOnRef.current && ( - <> - - - - -   Live streaming - - - {}} - /> - - since {liveTimestamp} - - - - )} - 0 - ? storedExplorerFields.selectedFields - : DEFAULT_EMPTY_EXPLORER_FIELDS - } - /> - - ​ - -
-
- - - - - ) : ( - - )} - - - +
+ {explorerData && !isEmpty(explorerData.jsonData) ? ( + + + + {/* */} + {countDistribution?.data && !isLiveTailOnRef.current && ( + <> + {}} + /> + { + const intervalOptionsIndex = timeIntervalOptions.findIndex( + (item) => item.value === selectedIntrv + ); + const intrv = selectedIntrv.replace(/^auto_/, ''); + getCountVisualizations(intrv); + selectedIntervalRef.current = timeIntervalOptions[intervalOptionsIndex]; + getPatterns(intrv, getErrorHandler('Error fetching patterns')); + }} + stateInterval={selectedIntervalRef.current?.value} + startTime={appLogEvents ? startTime : dateRange[0]} + endTime={appLogEvents ? endTime : dateRange[1]} + /> + + + )} + + + + + + + + + + + + + +
+

+ +

+
+ {isLiveTailOnRef.current && ( + <> + + + + +   Live streaming + + + {}} + /> + + since {liveTimestamp} + + + + )} + + 0 + ? storedExplorerFields.selectedFields + : DEFAULT_EMPTY_EXPLORER_FIELDS + } + /> + + ​ + +
+
+ + + + ) : ( + + )} +
); }, [ isPanelTextFieldInvalid, @@ -654,6 +607,7 @@ export const Explorer = ({ query, isLiveTailOnRef.current, isOverridingPattern, + isQueryRunning, ]); const visualizations: IVisualizationContainerProps = useMemo(() => { @@ -680,7 +634,7 @@ export const Explorer = ({ }; const explorerVis = useMemo(() => { - return ( + return explorerSearchMeta.datasources?.[0]?.type === 'DEFAULT_INDEX_PATTERNS' ? ( + ) : ( + ); - }, [query, curVisId, explorerFields, explorerVisualizations, explorerData, visualizations]); + }, [ + query, + curVisId, + explorerFields, + explorerVisualizations, + explorerData, + visualizations, + explorerSearchMeta.datasources, + ]); const contentTabs = [ { @@ -843,6 +807,8 @@ export const Explorer = ({ selectedCustomPanelOptions, ]); + // live tail + const liveTailLoop = async ( name: string, startingTime: string, @@ -944,49 +910,93 @@ export const Explorer = ({ uiSettingsService.get('theme:darkMode') && ' explorer-dark' }`} > - handleTimePickerChange(timeRange)} - selectedPanelName={selectedPanelNameRef.current} - selectedCustomPanelOptions={selectedCustomPanelOptions} - setSelectedPanelName={setSelectedPanelName} - setSelectedCustomPanelOptions={setSelectedCustomPanelOptions} - handleSavingObject={handleSavingObject} - isPanelTextFieldInvalid={isPanelTextFieldInvalid} - savedObjects={savedObjects} - showSavePanelOptionsList={isEqual(selectedContentTabId, TAB_CHART_ID)} - handleTimeRangePickerRefresh={handleTimeRangePickerRefresh} - isLiveTailPopoverOpen={isLiveTailPopoverOpen} - closeLiveTailPopover={() => setIsLiveTailPopoverOpen(false)} - popoverItems={popoverItems} - isLiveTailOn={isLiveTailOnRef.current} - selectedSubTabId={selectedContentTabId} - searchBarConfigs={searchBarConfigs} - getSuggestions={parseGetSuggestions} - onItemSelect={onItemSelect} - tabId={tabId} - baseQuery={appBaseQuery} - stopLive={stopLive} - setIsLiveTailPopoverOpen={setIsLiveTailPopoverOpen} - liveTailName={liveTailNameRef.current} - curVisId={curVisId} - setSubType={setSubType} - /> - tab.id === selectedContentTabId)} - onTabClick={(selectedTab: EuiTabbedContentTab) => handleContentTabClick(selectedTab)} - tabs={contentTabs} - size="s" - /> + + + + + + + +
+ 0 + ? storedExplorerFields + : explorerFields + } + setStoredExplorerFields={setStoredExplorerFields} + /> +
+ + + + + handleTimePickerChange(timeRange)} + selectedPanelName={selectedPanelNameRef.current} + selectedCustomPanelOptions={selectedCustomPanelOptions} + setSelectedPanelName={setSelectedPanelName} + setSelectedCustomPanelOptions={setSelectedCustomPanelOptions} + handleSavingObject={handleSavingObject} + isPanelTextFieldInvalid={isPanelTextFieldInvalid} + savedObjects={savedObjects} + showSavePanelOptionsList={isEqual(selectedContentTabId, TAB_CHART_ID)} + handleTimeRangePickerRefresh={handleTimeRangePickerRefresh} + isLiveTailPopoverOpen={isLiveTailPopoverOpen} + closeLiveTailPopover={() => setIsLiveTailPopoverOpen(false)} + popoverItems={popoverItems} + isLiveTailOn={isLiveTailOnRef.current} + selectedSubTabId={selectedContentTabId} + searchBarConfigs={searchBarConfigs} + getSuggestions={parseGetSuggestions} + onItemSelect={onItemSelect} + tabId={tabId} + baseQuery={appBaseQuery} + stopLive={stopLive} + setIsLiveTailPopoverOpen={setIsLiveTailPopoverOpen} + liveTailName={liveTailNameRef.current} + curVisId={curVisId} + setSubType={setSubType} + http={http} + setIsQueryRunning={setIsQueryRunning} + /> + {explorerSearchMeta.isPolling ? ( + + ) : ( + tab.id === selectedContentTabId)} + onTabClick={(selectedTab: EuiTabbedContentTab) => + handleContentTabClick(selectedTab) + } + tabs={contentTabs} + size="s" + /> + )} + +
); diff --git a/public/components/event_analytics/explorer/log_explorer.tsx b/public/components/event_analytics/explorer/log_explorer.tsx index ca8398863..78c0e90b4 100644 --- a/public/components/event_analytics/explorer/log_explorer.tsx +++ b/public/components/event_analytics/explorer/log_explorer.tsx @@ -14,7 +14,8 @@ import { TAB_CHART_ID, TAB_EVENT_ID, } from '../../../../common/constants/explorer'; -import { ILogExplorerProps } from '../../../../common/types/explorer'; +import { initializeTabData, removeTabData } from '../../application_analytics/helpers/utils'; +import { EmptyTabParams, ILogExplorerProps } from '../../../../common/types/explorer'; import { selectQueryResult } from '../redux/slices/query_result_slice'; import { selectQueries } from '../redux/slices/query_slice'; import { selectQueryTabs } from '../redux/slices/query_tab_slice'; @@ -31,6 +32,8 @@ const searchBarConfigs = { }, }; +const getExistingEmptyTab = ({ tabIds }: EmptyTabParams) => tabIds[0]; + export const LogExplorer = ({ pplService, dslService, @@ -38,10 +41,10 @@ export const LogExplorer = ({ timestampUtils, setToast, savedObjectId, - getExistingEmptyTab, notifications, http, queryManager, + dataSourcePluggables, }: ILogExplorerProps) => { const history = useHistory(); const routerContext = useContext(LogExplorerRouterContext); @@ -103,6 +106,7 @@ export const LogExplorer = ({ http={http} searchBarConfigs={searchBarConfigs} queryManager={queryManager} + dataSourcePluggables={dataSourcePluggables} /> ); diff --git a/public/components/event_analytics/explorer/sidebar/sidebar.tsx b/public/components/event_analytics/explorer/sidebar/sidebar.tsx index 29e8120d3..9ae6fb127 100644 --- a/public/components/event_analytics/explorer/sidebar/sidebar.tsx +++ b/public/components/event_analytics/explorer/sidebar/sidebar.tsx @@ -54,7 +54,6 @@ export const Sidebar = (props: ISidebarProps) => { storedExplorerFields, setStoredExplorerFields, } = props; - const dispatch = useDispatch(); const { tabId } = useContext(TabContext); const [showFields, setShowFields] = useState(false); @@ -241,9 +240,9 @@ export const Sidebar = (props: ISidebarProps) => { spacing="m" > {explorerData && - !isEmpty(explorerData.jsonData) && - storedExplorerFields.selectedFields && - storedExplorerFields.selectedFields.map((field, index) => { + !isEmpty(explorerData?.jsonData) && + storedExplorerFields?.selectedFields && + storedExplorerFields?.selectedFields.map((field, index) => { return ( { droppableId="" spacing="m" > - {storedExplorerFields.availableFields && - storedExplorerFields.availableFields + {storedExplorerFields?.availableFields && + storedExplorerFields?.availableFields .filter( (field) => searchTerm === '' || field.name.indexOf(searchTerm) !== -1 ) diff --git a/public/components/event_analytics/explorer/visualizations/direct_query_vis.tsx b/public/components/event_analytics/explorer/visualizations/direct_query_vis.tsx new file mode 100644 index 000000000..411d950ee --- /dev/null +++ b/public/components/event_analytics/explorer/visualizations/direct_query_vis.tsx @@ -0,0 +1,30 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import React from 'react'; +import { EuiFlexGroup, EuiFlexItem, EuiCallOut, EuiLink, EuiTitle } from '@elastic/eui'; + +export const DirectQueryVisualization = () => { + return ( + + + +

+ Index data to visualize. +

+ + + + +

Index data to visualize or select indexed data.

+ +

+ For external data only materialized views or covering indexes can be visualized. Ask your + administrator to create these indexes to visualize them. +

+ + + ); +}; diff --git a/public/components/event_analytics/explorer/visualizations/index.tsx b/public/components/event_analytics/explorer/visualizations/index.tsx index 41b6eab6a..f97db37f3 100644 --- a/public/components/event_analytics/explorer/visualizations/index.tsx +++ b/public/components/event_analytics/explorer/visualizations/index.tsx @@ -3,11 +3,9 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { isEmpty } from 'lodash'; import React from 'react'; import { EuiResizableContainer } from '@elastic/eui'; import { QueryManager } from 'common/query_manager'; -import { RAW_QUERY, SELECTED_TIMESTAMP } from '../../../../../common/constants/explorer'; import { IField, IQuery, @@ -16,9 +14,8 @@ import { } from '../../../../../common/types/explorer'; import { WorkspacePanel } from './workspace_panel'; import { ConfigPanel } from './config_panel'; -import { Sidebar } from '../sidebar'; import { DataConfigPanelItem } from './config_panel/config_panes/config_controls/data_configurations_panel'; -import { PPL_STATS_REGEX, VIS_CHART_TYPES } from '../../../../../common/constants/shared'; +import { VIS_CHART_TYPES } from '../../../../../common/constants/shared'; import { TreemapConfigPanelItem } from './config_panel/config_panes/config_controls/treemap_config_panel_item'; import { LogsViewConfigPanelItem } from './config_panel/config_panes/config_controls/logs_view_config_panel_item'; @@ -36,14 +33,11 @@ interface IExplorerVisualizationsProps { } export const ExplorerVisualizations = ({ - query, curVisId, setCurVisId, explorerVis, explorerFields, - explorerData, visualizations, - handleOverrideTimestamp, callback, queryManager, }: IExplorerVisualizationsProps) => { @@ -99,22 +93,7 @@ export const ExplorerVisualizations = ({ paddingSize="none" className="vis__leftPanel" > -
-
- -
+
{!isMarkDown && (
{ tabsState, } = props; const history = useHistory(); - const dispatch = useDispatch(); - const [searchQuery, setSearchQuery] = useState(''); const [selectedDateRange, setSelectedDateRange] = useState(['now-15m', 'now']); const [savedHistories, setSavedHistories] = useState([]); const [selectedHistories, setSelectedHistories] = useState([]); @@ -315,31 +313,6 @@ const EventAnalyticsHome = (props: IHomeProps) => { - - - - {}} - setEndTime={() => {}} - showSaveButton={false} - runButtonText="New Query" - getSuggestions={parseGetSuggestions} - onItemSelect={onItemSelect} - /> - - - - diff --git a/public/components/event_analytics/hooks/use_fetch_direct_events.ts b/public/components/event_analytics/hooks/use_fetch_direct_events.ts new file mode 100644 index 000000000..9b92026f9 --- /dev/null +++ b/public/components/event_analytics/hooks/use_fetch_direct_events.ts @@ -0,0 +1,228 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { useState, useRef } from 'react'; +import { batch } from 'react-redux'; +import { isEmpty } from 'lodash'; +import { useDispatch, useSelector } from 'react-redux'; +import { IField } from 'common/types/explorer'; +import { + FINAL_QUERY, + SELECTED_FIELDS, + UNSELECTED_FIELDS, + AVAILABLE_FIELDS, + QUERIED_FIELDS, +} from '../../../../common/constants/explorer'; +import { fetchSuccess, reset as queryResultReset } from '../redux/slices/query_result_slice'; +import { reset as patternsReset } from '../redux/slices/patterns_slice'; +import { selectQueries } from '../redux/slices/query_slice'; +import { reset as visualizationReset } from '../redux/slices/visualization_slice'; +import { updateFields, sortFields, selectFields } from '../redux/slices/field_slice'; +import PPLService from '../../../services/requests/ppl'; +import { PPL_STATS_REGEX } from '../../../../common/constants/shared'; + +interface IFetchEventsParams { + pplService: PPLService; + requestParams: { tabId: string }; +} + +export const useFetchDirectEvents = ({ pplService, requestParams }: IFetchEventsParams) => { + const dispatch = useDispatch(); + const [isEventsLoading, setIsEventsLoading] = useState(false); + const queries = useSelector(selectQueries); + const fields = useSelector(selectFields); + const [response, setResponse] = useState(); + const queriesRef = useRef(); + const fieldsRef = useRef(); + const responseRef = useRef(); + queriesRef.current = queries; + fieldsRef.current = fields; + responseRef.current = response; + + const fetchEvents = ( + { query }: { query: string }, + format: string, + handler: (res: any) => unknown, + errorHandler?: (error: any) => void + ) => { + setIsEventsLoading(true); + return pplService + .fetch({ query, format }, errorHandler) + .then((res: any) => handler(res)) + .catch((err: any) => { + console.error(err); + throw err; + }) + .finally(() => setIsEventsLoading(false)); + }; + + const addSchemaRowMapping = (queryResult) => { + const pplRes = queryResult; + + const data: any[] = []; + + _.forEach(pplRes.datarows, (row) => { + const record: any = {}; + + for (let i = 0; i < pplRes.schema.length; i++) { + const cur = pplRes.schema[i]; + + if (typeof row[i] === 'object') { + record[cur.name] = JSON.stringify(row[i]); + } else if (typeof row[i] === 'boolean') { + record[cur.name] = row[i].toString(); + } else { + record[cur.name] = row[i]; + } + } + + data.push(record); + }); + return { + ...queryResult, + jsonData: data, + }; + }; + + const dispatchOnGettingHis = (res: any, query: string) => { + const processedRes = addSchemaRowMapping(res); + const selectedFields: string[] = fieldsRef.current![requestParams.tabId][SELECTED_FIELDS].map( + (field: IField) => field.name + ); + batch(() => { + dispatch( + queryResultReset({ + tabId: requestParams.tabId, + }) + ); + dispatch( + fetchSuccess({ + tabId: requestParams.tabId, + data: { + ...processedRes, + }, + }) + ); + dispatch( + updateFields({ + tabId: requestParams.tabId, + data: { + [UNSELECTED_FIELDS]: processedRes?.schema ? [...processedRes.schema] : [], + [QUERIED_FIELDS]: query.match(PPL_STATS_REGEX) ? [...processedRes.schema] : [], // when query contains stats, need populate this + [AVAILABLE_FIELDS]: processedRes?.schema ? [...processedRes.schema] : [], + [SELECTED_FIELDS]: [], + }, + }) + ); + dispatch( + sortFields({ + tabId: requestParams.tabId, + data: [AVAILABLE_FIELDS, UNSELECTED_FIELDS], + }) + ); + dispatch( + visualizationReset({ + tabId: requestParams.tabId, + }) + ); + }); + }; + + const dispatchOnNoHis = (res: any) => { + setResponse(res); + batch(() => { + dispatch( + queryResultReset({ + tabId: requestParams.tabId, + }) + ); + dispatch( + updateFields({ + tabId: requestParams.tabId, + data: { + [SELECTED_FIELDS]: [], + [UNSELECTED_FIELDS]: [], + [QUERIED_FIELDS]: [], + [AVAILABLE_FIELDS]: res?.schema ? [...res.schema] : [], + }, + }) + ); + dispatch( + sortFields({ + tabId: requestParams.tabId, + data: [AVAILABLE_FIELDS], + }) + ); + dispatch( + visualizationReset({ + tabId: requestParams.tabId, + }) + ); + dispatch( + patternsReset({ + tabId: requestParams.tabId, + }) + ); + }); + }; + + const getLiveTail = (query: string = '', errorHandler?: (error: any) => void) => { + const cur = queriesRef.current; + const searchQuery = isEmpty(query) ? cur![requestParams.tabId][FINAL_QUERY] : query; + fetchEvents( + { query: searchQuery }, + 'jdbc', + (res: any) => { + if (!isEmpty(res.jsonData)) { + if (!isEmpty(responseRef.current)) { + res.jsonData = res.jsonData.concat(responseRef.current.jsonData); + res.datarows = res.datarows.concat(responseRef.current.datarows); + res.total = res.total + responseRef.current.total; + res.size = res.size + responseRef.current.size; + } + dispatchOnGettingHis(res, searchQuery); + } + if (isEmpty(res.jsonData) && isEmpty(responseRef.current)) { + dispatchOnNoHis(res); + } + }, + errorHandler + ); + }; + + const getEvents = (query: string = '', errorHandler?: (error: any) => void) => { + if (isEmpty(query)) return; + return dispatchOnGettingHis(res, ''); + }; + + const getAvailableFields = (query: string) => { + fetchEvents({ query }, 'jdbc', (res: any) => { + batch(() => { + dispatch( + updateFields({ + tabId: requestParams.tabId, + data: { + [AVAILABLE_FIELDS]: res?.schema ? [...res.schema] : [], + }, + }) + ); + dispatch( + sortFields({ + tabId: requestParams.tabId, + data: [AVAILABLE_FIELDS, UNSELECTED_FIELDS], + }) + ); + }); + }); + }; + + return { + isEventsLoading, + getLiveTail, + getEvents, + getAvailableFields, + fetchEvents, + }; +}; diff --git a/public/components/event_analytics/hooks/use_fetch_events.ts b/public/components/event_analytics/hooks/use_fetch_events.ts index cfd229eb0..16a77c436 100644 --- a/public/components/event_analytics/hooks/use_fetch_events.ts +++ b/public/components/event_analytics/hooks/use_fetch_events.ts @@ -58,11 +58,37 @@ export const useFetchEvents = ({ pplService, requestParams }: IFetchEventsParams .finally(() => setIsEventsLoading(false)); }; + const addSchemaRowMapping = (queryResult) => { + const pplRes = queryResult; + + const data: any[] = []; + + _.forEach(pplRes.datarows, (row) => { + const record: any = {}; + + for (let i = 0; i < pplRes.schema.length; i++) { + const cur = pplRes.schema[i]; + + if (typeof row[i] === 'object') { + record[cur.name] = JSON.stringify(row[i]); + } else if (typeof row[i] === 'boolean') { + record[cur.name] = row[i].toString(); + } else { + record[cur.name] = row[i]; + } + } + + data.push(record); + }); + return { + ...queryResult, + jsonData: data, + }; + }; + const dispatchOnGettingHis = (res: any, query: string) => { - const selectedFields: string[] = fieldsRef.current![requestParams.tabId][SELECTED_FIELDS].map( - (field: IField) => field.name - ); - setResponse(res); + const processedRes = addSchemaRowMapping(res); + setResponse(processedRes); batch(() => { dispatch( queryResultReset({ @@ -73,7 +99,7 @@ export const useFetchEvents = ({ pplService, requestParams }: IFetchEventsParams fetchSuccess({ tabId: requestParams.tabId, data: { - ...res, + ...processedRes, }, }) ); @@ -81,9 +107,9 @@ export const useFetchEvents = ({ pplService, requestParams }: IFetchEventsParams updateFields({ tabId: requestParams.tabId, data: { - [UNSELECTED_FIELDS]: res?.schema ? [...res.schema] : [], - [QUERIED_FIELDS]: query.match(PPL_STATS_REGEX) ? [...res.schema] : [], // when query contains stats, need populate this - [AVAILABLE_FIELDS]: res?.schema ? [...res.schema] : [], + [UNSELECTED_FIELDS]: processedRes?.schema ? [...processedRes.schema] : [], + [QUERIED_FIELDS]: query.match(PPL_STATS_REGEX) ? [...processedRes.schema] : [], // when query contains stats, need populate this + [AVAILABLE_FIELDS]: processedRes?.schema ? [...processedRes.schema] : [], [SELECTED_FIELDS]: [], }, }) @@ -165,6 +191,7 @@ export const useFetchEvents = ({ pplService, requestParams }: IFetchEventsParams }; const getEvents = (query: string = '', errorHandler?: (error: any) => void) => { + if (isEmpty(query)) return; const cur = queriesRef.current; const searchQuery = isEmpty(query) ? cur![requestParams.tabId][FINAL_QUERY] : query; fetchEvents( @@ -173,6 +200,8 @@ export const useFetchEvents = ({ pplService, requestParams }: IFetchEventsParams (res: any) => { if (!isEmpty(res.jsonData)) { return dispatchOnGettingHis(res, searchQuery); + } else if (!isEmpty(res.data?.resp)) { + return dispatchOnGettingHis(JSON.parse(res.data?.resp), searchQuery); } // when no hits and needs to get available fields to override default timestamp dispatchOnNoHis(res); @@ -208,5 +237,6 @@ export const useFetchEvents = ({ pplService, requestParams }: IFetchEventsParams getEvents, getAvailableFields, fetchEvents, + dispatchOnGettingHis, }; }; diff --git a/public/components/event_analytics/index.tsx b/public/components/event_analytics/index.tsx index e5714caac..959a96b73 100644 --- a/public/components/event_analytics/index.tsx +++ b/public/components/event_analytics/index.tsx @@ -81,9 +81,9 @@ export const EventAnalytics = ({ timestampUtils={timestampUtils} http={http} setToast={setToast} - getExistingEmptyTab={getExistingEmptyTab} notifications={notifications} queryManager={queryManager} + dataSourcePluggables={props.dataSourcePluggables} /> ); @@ -108,7 +108,7 @@ export const EventAnalytics = ({ dslService={dslService} pplService={pplService} setToast={setToast} - getExistingEmptyTab={getExistingEmptyTab} + dataSourcePluggables={props.dataSourcePluggables} /> ); }} diff --git a/public/components/event_analytics/redux/slices/count_distribution_slice.ts b/public/components/event_analytics/redux/slices/count_distribution_slice.ts index 5f0ec00fb..59aa1e03f 100644 --- a/public/components/event_analytics/redux/slices/count_distribution_slice.ts +++ b/public/components/event_analytics/redux/slices/count_distribution_slice.ts @@ -20,11 +20,14 @@ export const countDistributionSlice = createSlice({ ...payload.data, }; }, + reset: (state, { payload }) => { + state[payload.tabId] = {}; + }, }, extraReducers: (builder) => {}, }); -export const { render } = countDistributionSlice.actions; +export const { render, reset } = countDistributionSlice.actions; export const selectCountDistribution = createSelector( (state) => state.countDistribution, diff --git a/public/components/event_analytics/redux/slices/query_slice.ts b/public/components/event_analytics/redux/slices/query_slice.ts index ec7a47527..8267d72f9 100644 --- a/public/components/event_analytics/redux/slices/query_slice.ts +++ b/public/components/event_analytics/redux/slices/query_slice.ts @@ -71,6 +71,11 @@ export const queriesSlice = createSlice({ remove: (state, { payload }) => { delete state[payload.tabId]; }, + reset: (state, { payload }) => { + state[payload.tabId] = { + ...initialQueryState, + }; + }, }, extraReducers: (builder) => {}, }); diff --git a/public/components/event_analytics/redux/slices/search_meta_data_slice.ts b/public/components/event_analytics/redux/slices/search_meta_data_slice.ts new file mode 100644 index 000000000..eee7083fa --- /dev/null +++ b/public/components/event_analytics/redux/slices/search_meta_data_slice.ts @@ -0,0 +1,47 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { createSlice, createSelector } from '@reduxjs/toolkit'; +import { initialTabId } from '../../../../framework/redux/store/shared_state'; +import { REDUX_EXPL_SLICE_SEARCH_META_DATA } from '../../../../../common/constants/explorer'; + +const initialState = { + [initialTabId]: { + lang: 'PPL', + datasources: [], + isPolling: false, + }, +}; + +export const searchMetaDataSlice = createSlice({ + name: REDUX_EXPL_SLICE_SEARCH_META_DATA, + initialState, + reducers: { + update: (state, { payload }) => { + state[payload.tabId] = { + ...state[payload.tabId], + ...payload.data, + }; + }, + reset: (state, { payload }) => { + state[payload.tabId] = {}; + }, + init: (state, { payload }) => { + state[payload.tabId] = {}; + }, + remove: (state, { payload }) => { + delete state[payload.tabId]; + }, + }, +}); + +export const { update, remove, init } = searchMetaDataSlice.actions; + +export const selectSearchMetaData = createSelector( + (state) => state.searchMetadata, + (searchMetadata) => searchMetadata +); + +export const searchMetaDataSliceReducer = searchMetaDataSlice.reducer; diff --git a/public/components/hooks/index.ts b/public/components/hooks/index.ts new file mode 100644 index 000000000..d3c599c18 --- /dev/null +++ b/public/components/hooks/index.ts @@ -0,0 +1,6 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +export { usePolling } from './use_polling'; diff --git a/public/components/hooks/use_direct_query_search.ts b/public/components/hooks/use_direct_query_search.ts new file mode 100644 index 000000000..a850c1690 --- /dev/null +++ b/public/components/hooks/use_direct_query_search.ts @@ -0,0 +1,4 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ diff --git a/public/components/hooks/use_polling.ts b/public/components/hooks/use_polling.ts new file mode 100644 index 000000000..42fe5b9d3 --- /dev/null +++ b/public/components/hooks/use_polling.ts @@ -0,0 +1,56 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { useState, useRef } from 'react'; + +type FetchFunction = (params?: P) => Promise; + +interface UsePollingReturn { + data: T | null; + loading: boolean; + error: Error | null; + startPolling: (params?: any) => void; + stopPolling: () => void; +} + +export function usePolling( + fetchFunction: FetchFunction, + interval: number = 5000 +): UsePollingReturn { + const [data, setData] = useState(null); + const [error, setError] = useState(null); + const [loading, setLoading] = useState(true); + const intervalRef = useRef(undefined); + + const shouldPoll = useRef(false); + + const startPolling = (params?: P) => { + shouldPoll.current = true; + const intervalId = setInterval(() => { + if (shouldPoll.current) { + fetchData(params); + } + }, interval); + intervalRef.current = intervalId; + }; + + const stopPolling = () => { + shouldPoll.current = false; + clearInterval(intervalRef.current); + }; + + const fetchData = async (params?: P) => { + try { + const result = await fetchFunction(params); + setData(result); + } catch (err) { + setError(err); + } finally { + setLoading(false); + } + }; + + return { data, loading, error, startPolling, stopPolling }; +} diff --git a/public/components/index.tsx b/public/components/index.tsx index 819175297..1058af43a 100644 --- a/public/components/index.tsx +++ b/public/components/index.tsx @@ -7,7 +7,7 @@ import React from 'react'; import ReactDOM from 'react-dom'; import { QueryManager } from 'common/query_manager'; import { AppMountParameters, CoreStart } from '../../../../src/core/public'; -import { AppPluginStartDependencies } from '../types'; +import { AppPluginStartDependencies, SetupDependencies } from '../types'; import { App } from './app'; export const Observability = ( @@ -19,7 +19,8 @@ export const Observability = ( savedObjects: any, timestampUtils: any, queryManager: QueryManager, - startPage: string + startPage: string, + dataSourcePluggables ) => { ReactDOM.render( , AppMountParametersProp.element ); diff --git a/public/framework/datasource_pluggables/datasource_pluggable.ts b/public/framework/datasource_pluggables/datasource_pluggable.ts new file mode 100644 index 000000000..d5212d2e1 --- /dev/null +++ b/public/framework/datasource_pluggables/datasource_pluggable.ts @@ -0,0 +1,29 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { IDataSourceComponentSet, IDataSourcePluggableComponents } from './types'; + +export class DataSourcePluggable { + private components: IDataSourcePluggableComponents = {}; + + public addVariationSet( + variationKey: string, + variationValue: string, + componentSet: IDataSourceComponentSet + ) { + if (!this.components[variationKey]) { + this.components[variationKey] = {}; + } + this.components[variationKey][variationValue] = componentSet; + return this; + } + + public getComponentSetForVariation( + variationKey: string, + variationValue: string + ): IDataSourceComponentSet | undefined { + return this.components[variationKey]?.[variationValue]; + } +} diff --git a/public/framework/datasource_pluggables/types.ts b/public/framework/datasource_pluggables/types.ts new file mode 100644 index 000000000..2c3a37963 --- /dev/null +++ b/public/framework/datasource_pluggables/types.ts @@ -0,0 +1,22 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { IDataFetcher } from '../../services/data_fetchers/fetch_interface'; + +export interface IDataSourceComponentSet { + ui: { + QueryEditor: React.ReactNode; + ConfigEditor: React.ReactNode; + SidePanel: React.ReactNode; + }; + services: { + data_fetcher: IDataFetcher; + }; +} + +export interface IDataSourcePluggableComponents { + languages?: Record; + // Other variation keys can be added in the future +} diff --git a/public/framework/datasources/s3_datasource.ts b/public/framework/datasources/s3_datasource.ts new file mode 100644 index 000000000..ef95f0955 --- /dev/null +++ b/public/framework/datasources/s3_datasource.ts @@ -0,0 +1,30 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { DataSource } from '../../../../../src/plugins/data/public'; + +interface DataSourceConfig { + name: string; + type: string; + metadata: any; +} + +export class S3DataSource extends DataSource { + constructor({ name, type, metadata }: DataSourceConfig) { + super(name, type, metadata); + } + + async getDataSet(dataSetParams?: any) { + return [this.getName()]; + } + + async testConnection(): Promise { + throw new Error('This operation is not supported for this class.'); + } + + async runQuery(queryParams: any) { + return null; + } +} diff --git a/public/framework/redux/reducers/index.ts b/public/framework/redux/reducers/index.ts index d392b8265..6fad432d4 100644 --- a/public/framework/redux/reducers/index.ts +++ b/public/framework/redux/reducers/index.ts @@ -15,6 +15,7 @@ import { explorerVisualizationConfigReducer } from '../../../components/event_an import { patternsReducer } from '../../../components/event_analytics/redux/slices/patterns_slice'; import { metricsReducers } from '../../../components/metrics/redux/slices/metrics_slice'; import { panelReducer } from '../../../components/custom_panels/redux/panel_slice'; +import { searchMetaDataSliceReducer } from '../../../components/event_analytics/redux/slices/search_meta_data_slice'; const combinedReducer = combineReducers({ // explorer reducers @@ -28,6 +29,7 @@ const combinedReducer = combineReducers({ patterns: patternsReducer, metrics: metricsReducers, customPanel: panelReducer, + searchMetadata: searchMetaDataSliceReducer, }); export type RootState = ReturnType; diff --git a/public/plugin.ts b/public/plugin.ts index c402e8a0c..9ac1fe588 100644 --- a/public/plugin.ts +++ b/public/plugin.ts @@ -4,7 +4,6 @@ */ import './index.scss'; - import { i18n } from '@osd/i18n'; import { AppCategory, @@ -39,6 +38,8 @@ import { observabilityIntegrationsTitle, observabilityIntegrationsPluginOrder, observabilityPluginOrder, + DATACONNECTIONS_BASE, + S3_DATASOURCE_TYPE, observabilityDataConnectionsID, observabilityDataConnectionsPluginOrder, observabilityDataConnectionsTitle, @@ -55,7 +56,6 @@ import { convertLegacyNotebooksUrl } from './components/notebooks/components/hel import { convertLegacyTraceAnalyticsUrl } from './components/trace_analytics/components/common/legacy_route_helpers'; import { SavedObject } from '../../../src/core/public'; import { coreRefs } from './framework/core_refs'; - import { OBSERVABILITY_EMBEDDABLE, OBSERVABILITY_EMBEDDABLE_DESCRIPTION, @@ -75,6 +75,10 @@ import { ObservabilityStart, SetupDependencies, } from './types'; +import { S3DataSource } from './framework/datasources/s3_datasource'; +import { DataSourcePluggable } from './framework/datasource_pluggables/datasource_pluggable'; +import { DirectSearch } from './components/common/search/sql_search'; +import { Search } from './components/common/search/search'; export class ObservabilityPlugin implements @@ -124,13 +128,63 @@ export class ObservabilityPlugin }, }); + // Adding a variation entails associating a key-value pair, where a change in the key results in + // a switch of UI/services to its corresponding context. In the following cases, for an S3 datasource, + // selecting SQL will render SQL-specific UI components or services, while selecting PPL will + // render a set of UI components or services specific to PPL. + const openSearchLocalDataSourcePluggable = new DataSourcePluggable().addVariationSet( + 'languages', + 'PPL', + { + ui: { + QueryEditor: null, + ConfigEditor: null, + SidePanel: null, + SearchBar: Search, + }, + services: {}, + } + ); + + const s3DataSourcePluggable = new DataSourcePluggable() + .addVariationSet('languages', 'SQL', { + ui: { + QueryEditor: null, + ConfigEditor: null, + SidePanel: null, + SearchBar: DirectSearch, + }, + services: { + data_fetcher: null, + }, + }) + .addVariationSet('languages', 'PPL', { + ui: { + QueryEditor: null, + ConfigEditor: null, + SidePanel: null, + SearchBar: DirectSearch, + }, + services: { + data_fetcher: null, + }, + }); + + // below datasource types is referencing: + // https://github.com/opensearch-project/sql/blob/feature/job-apis/core/src/main/java/org/opensearch/sql/datasource/model/DataSourceType.java + const dataSourcePluggables = { + DEFAULT_INDEX_PATTERNS: openSearchLocalDataSourcePluggable, + spark: s3DataSourcePluggable, + s3glue: s3DataSourcePluggable, + // prometheus: openSearchLocalDataSourcePluggable + }; + const appMountWithStartPage = (startPage: string) => async (params: AppMountParameters) => { const { Observability } = await import('./components/index'); const [coreStart, depsStart] = await core.getStartServices(); const dslService = new DSLService(coreStart.http); const savedObjects = new SavedObjects(coreStart.http); const timestampUtils = new TimestampUtils(dslService, pplService); - return Observability( coreStart, depsStart as AppPluginStartDependencies, @@ -140,7 +194,8 @@ export class ObservabilityPlugin savedObjects, timestampUtils, qm, - startPage + startPage, + dataSourcePluggables // just pass down for now due to time constraint, later may better expose this as context ); }; @@ -255,7 +310,7 @@ export class ObservabilityPlugin return {}; } - public start(core: CoreStart): ObservabilityStart { + public start(core: CoreStart, startDeps: AppPluginStartDependencies): ObservabilityStart { const pplService: PPLService = new PPLService(core.http); coreRefs.http = core.http; @@ -263,8 +318,25 @@ export class ObservabilityPlugin coreRefs.pplService = pplService; coreRefs.toasts = core.notifications.toasts; coreRefs.chrome = core.chrome; + coreRefs.dataSources = startDeps.data.dataSources; coreRefs.application = core.application; + const { dataSourceService, dataSourceFactory } = startDeps.data.dataSources; + + // register all s3 datasources + dataSourceFactory.registerDataSourceType(S3_DATASOURCE_TYPE, S3DataSource); + core.http.get(`${DATACONNECTIONS_BASE}`).then((s3DataSources) => { + s3DataSources.map((s3ds) => { + dataSourceService.registerDataSource( + dataSourceFactory.getDataSourceInstance(S3_DATASOURCE_TYPE, { + name: s3ds.name, + type: s3ds.connector.toLowerCase(), + metadata: s3ds, + }) + ); + }); + }); + return {}; } diff --git a/public/services/data_fetchers/sql/sql_data_fetcher.ts b/public/services/data_fetchers/sql/sql_data_fetcher.ts new file mode 100644 index 000000000..6a1fc967c --- /dev/null +++ b/public/services/data_fetchers/sql/sql_data_fetcher.ts @@ -0,0 +1,45 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { isEmpty } from 'lodash'; +import { IDefaultTimestampState, IQuery } from '../../../../common/types/explorer'; +import { IDataFetcher } from '../fetch_interface'; +import { DataFetcherBase } from '../fetcher_base'; +import { + buildRawQuery, + composeFinalQuery, + getIndexPatternFromRawQuery, +} from '../../../../common/utils'; +import { + FILTERED_PATTERN, + PATTERNS_REGEX, + PATTERN_REGEX, + RAW_QUERY, + SELECTED_DATE_RANGE, + SELECTED_PATTERN_FIELD, + SELECTED_TIMESTAMP, + TAB_CHART_ID, +} from '../../../../common/constants/explorer'; +import { PPL_BASE, PPL_SEARCH, PPL_STATS_REGEX } from '../../../../common/constants/shared'; +import { CoreStart } from '../../../../../../src/core/public'; +import { useFetchEvents } from '../../../components/event_analytics/hooks'; +import { SQLService } from '../../../services/requests/sql'; + +export class SQLDataFetcher extends DataFetcherBase implements IDataFetcher { + constructor(private readonly http: CoreStart['http']) { + super(); + } + + async search(query: string, callback) { + callback(query); + + const sqlService = new SQLService(this.http); + return sqlService.fetch({ + query, + lang: 'sql', + datasource: '', + }); + } +} diff --git a/public/services/requests/sql.ts b/public/services/requests/sql.ts new file mode 100644 index 000000000..e87d4d726 --- /dev/null +++ b/public/services/requests/sql.ts @@ -0,0 +1,41 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { CoreStart } from '../../../../../src/core/public'; +import { PPL_BASE, PPL_SEARCH } from '../../../common/constants/shared'; + +export class SQLService { + private http; + constructor(http: CoreStart['http']) { + this.http = http; + } + + fetch = async ( + params: { + query: string; + lang: string; + datasource: string; + }, + errorHandler?: (error: any) => void + ) => { + return this.http + .post('/api/observability/query/jobs', { + body: JSON.stringify(params), + }) + .catch((error) => { + console.error('fetch error: ', error.body); + if (errorHandler) errorHandler(error); + throw error; + }); + }; + + fetchWithJobId = async (params: { queryId: string }, errorHandler?: (error: any) => void) => { + return this.http.get(`/api/observability/query/jobs/${params.queryId}`).catch((error) => { + console.error('fetch error: ', error.body); + if (errorHandler) errorHandler(error); + throw error; + }); + }; +} diff --git a/public/types.ts b/public/types.ts index 75afb133b..4b6cd96a4 100644 --- a/public/types.ts +++ b/public/types.ts @@ -3,9 +3,10 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { CoreStart } from '../../../src/core/public'; import { SavedObjectsClient } from '../../../src/core/server'; import { DashboardStart } from '../../../src/plugins/dashboard/public'; -import { DataPublicPluginSetup } from '../../../src/plugins/data/public'; +import { DataPublicPluginSetup, DataPublicPluginStart } from '../../../src/plugins/data/public'; import { EmbeddableSetup, EmbeddableStart } from '../../../src/plugins/embeddable/public'; import { ManagementOverViewPluginSetup } from '../../../src/plugins/management_overview/public'; import { NavigationPublicPluginStart } from '../../../src/plugins/navigation/public'; @@ -17,6 +18,7 @@ export interface AppPluginStartDependencies { embeddable: EmbeddableStart; dashboard: DashboardStart; savedObjectsClient: SavedObjectsClient; + data: DataPublicPluginStart; } export interface SetupDependencies { diff --git a/server/adaptors/opensearch_observability_plugin.ts b/server/adaptors/opensearch_observability_plugin.ts index 10137b33c..012841fc4 100644 --- a/server/adaptors/opensearch_observability_plugin.ts +++ b/server/adaptors/opensearch_observability_plugin.ts @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { OPENSEARCH_DATASOURCES_API, OPENSEARCH_PANELS_API } from '../../common/constants/shared'; +import { JOBS_ENDPOINT_BASE, OPENSEARCH_DATASOURCES_API, OPENSEARCH_PANELS_API } from '../../common/constants/shared'; export function OpenSearchObservabilityPlugin(Client: any, config: any, components: any) { const clientAction = components.clientAction.factory; @@ -116,4 +116,25 @@ export function OpenSearchObservabilityPlugin(Client: any, config: any, componen }, method: 'DELETE', }); + + observability.getJobStatus = clientAction({ + url: { + fmt: `${JOBS_ENDPOINT_BASE}/<%=queryId%>`, + req: { + queryId: { + type: 'string', + required: true, + }, + }, + }, + method: 'GET', + }); + + observability.runDirectQuery = clientAction({ + url: { + fmt: `${JOBS_ENDPOINT_BASE}`, + }, + method: 'POST', + needBody: true, + }); } diff --git a/server/routes/sql.ts b/server/routes/sql.ts new file mode 100644 index 000000000..725b5ca84 --- /dev/null +++ b/server/routes/sql.ts @@ -0,0 +1,38 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { schema } from '@osd/config-schema'; +import { IRouter, IOpenSearchDashboardsResponse, ResponseError } from '../../../../src/core/server'; +import PPLFacet from '../services/facets/ppl_facet'; +import { PPL_BASE, PPL_SEARCH } from '../../common/constants/shared'; + +export function registerSqlRoute({ router, facet }: { router: IRouter; facet: PPLFacet }) { + router.post( + { + path: `/api/sql/search`, + validate: { + body: schema.object({ + query: schema.string(), + format: schema.string(), + }), + }, + }, + async (context, req, res): Promise> => { + const queryRes: any = await facet.describeQuery(req); + if (queryRes.success) { + const result: any = { + body: { + ...queryRes.data, + }, + }; + return res.ok(result); + } + return res.custom({ + statusCode: queryRes.data.statusCode || queryRes.data.status || 500, + body: queryRes.data.body || queryRes.data.message || '', + }); + } + ); +}