From 03aec60eaf114e33e3223ffe7934c39df14529d6 Mon Sep 17 00:00:00 2001 From: "Qingyang(Abby) Hu" Date: Fri, 6 Jan 2023 12:02:11 -0800 Subject: [PATCH] [Vis Builder] Add app filter and query persistence without using state container (#3100) * connect without container Signed-off-by: abbyhu2000 * Query and filter persistence working Signed-off-by: abbyhu2000 * Rebase and changelog Signed-off-by: abbyhu2000 * Simplify filter management Signed-off-by: abbyhu2000 * change function name Signed-off-by: abbyhu2000 * add unit test for function connect storage to query Signed-off-by: abbyhu2000 * changelog change Signed-off-by: abbyhu2000 * add comments and functional documentation in readme Signed-off-by: abbyhu2000 * add more documentation in data persistence readme about vis builder persistence Signed-off-by: abbyhu2000 * error handling Signed-off-by: abbyhu2000 * add function definition in data plugin api doc Signed-off-by: abbyhu2000 * document and comments Signed-off-by: abbyhu2000 Signed-off-by: abbyhu2000 Signed-off-by: Arpit Bandejiya --- CHANGELOG.md | 3 +- docs/plugins/data_persistence.md | 2 + src/plugins/data/public/index.ts | 1 + src/plugins/data/public/public.api.md | 14 ++ .../data/public/query/state_sync/README.md | 23 ++++ .../state_sync/connect_to_query_state.test.ts | 123 +++++++++++++++++- .../state_sync/connect_to_query_state.ts | 103 ++++++++++++++- .../data/public/query/state_sync/index.ts | 2 +- .../public/application/components/top_nav.tsx | 5 + 9 files changed, 270 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4e94748ce6b..755ab6460f1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,10 +49,10 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - [I18n] Register ru, ru-RU locale ([#2817](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2817)) - Add yarn opensearch arg to setup plugin dependencies ([#2544](https://github.com/opensearch-project/OpenSearch-Dashboards/issues/2544)) - [Multi DataSource] Test the connection to an external data source when creating or updating ([#2973](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2973)) -- [Doc] Add current plugin persistence implementation readme ([#3081](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3081)) - [Table Visualization] Refactor table visualization using React and DataGrid component ([#2863](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2863)) - [Vis Builder] Add redux store persistence ([#3088](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3088)) - [Multi DataSource] Improve test connection ([#3110](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3110)) +- [Vis Builder] Add app filter and query persistence without using state container ([#3100](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3100)) - [Optimizer] Increase timeout waiting for the exiting of an optimizer worker ([#3193](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3193)) ### 🐛 Bug Fixes @@ -105,6 +105,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Add `current-usage.md` and more details to `README.md` of `charts` plugin ([#2695](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2695)) - [Doc] Add readme for global query persistence ([#3001](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3001)) - Updates NOTICE file, adds validation to GitHub CI ([#3051](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3051)) +- [Doc] Add current plugin persistence implementation readme ([#3081](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3081)) ### 🛠 Maintenance diff --git a/docs/plugins/data_persistence.md b/docs/plugins/data_persistence.md index 84e71c9c3242..b4141e8a57a6 100644 --- a/docs/plugins/data_persistence.md +++ b/docs/plugins/data_persistence.md @@ -196,6 +196,8 @@ There are two types for data persistence: 4. In `useEditorUpdates()`, we use the saved appState to load the visualize editor +5. We can also choose to do state sync without using state container by directing hooking up the state managers with the URL data storage. For example, we implemented state management in Vis Builder with redux store, while the global state persistence implementation is the same, we implemented app state persistence without using any state containers or state syncing utils. For the actual visual data, we directly hook up redux store with `OsdUrlStateStorage` to sync the values by using `saveReduxState()` and `loadReduxState()`. For app filter and query, since they are part of the app states but not part of the redux store, we directly hook up their state managers from data plugin with `OsdUrlStateStorage` and `connectStorageToQueryState()`. + # Refresh When we refresh the page, both app state and global state should persist: diff --git a/src/plugins/data/public/index.ts b/src/plugins/data/public/index.ts index 8b495a2a19bf..3f96955e22b6 100644 --- a/src/plugins/data/public/index.ts +++ b/src/plugins/data/public/index.ts @@ -444,6 +444,7 @@ export { Filter, Query, RefreshInterval, TimeRange } from '../common'; export { createSavedQueryService, + connectStorageToQueryState, connectToQueryState, syncQueryStateWithUrl, QueryState, diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md index 615ca3c2e0dd..4651f4a2b700 100644 --- a/src/plugins/data/public/public.api.md +++ b/src/plugins/data/public/public.api.md @@ -368,6 +368,20 @@ export enum BUCKET_TYPES { // @public export const castOpenSearchToOsdFieldTypeName: (opensearchType: OPENSEARCH_FIELD_TYPES | string) => OSD_FIELD_TYPES; +// @public +export const connectStorageToQueryState = ( + { + filterManager, + queryString, + state$, + }: Pick, + OsdUrlStateStorage: IOsdUrlStateStorage, + syncConfig: { + filters: FilterStateStore; + query: boolean; + } +) => () => void; + // Warning: (ae-forgotten-export) The symbol "QuerySetup" needs to be exported by the entry point index.d.ts // Warning: (ae-forgotten-export) The symbol "BaseStateContainer" needs to be exported by the entry point index.d.ts // Warning: (ae-missing-release-tag) "connectToQueryState" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) diff --git a/src/plugins/data/public/query/state_sync/README.md b/src/plugins/data/public/query/state_sync/README.md index 6b9b15810057..f54688afccff 100644 --- a/src/plugins/data/public/query/state_sync/README.md +++ b/src/plugins/data/public/query/state_sync/README.md @@ -1,3 +1,26 @@ # Query state syncing utilities Set of helpers to connect data services to state containers and state syncing utilities + +# Connect to query state + +This set of functions help sync state storage and state container with query managers. + +1. `connectStorageToQueryState()` + - This function take three input parameters: query state managers, `OsdUrlStateStorage`, and the two configs that it helps syncing, `app filters` and `query` + - If `OsdUrlStateStorage` is empty, then we initialize the `OsdUrlStateStorage` using the default app filter and query by calling `getDefaultQuery()` and `getAppFilters()` + - If the current query state and filter state differentiate from the url state storage, we update the filter and query values using state managers for filter and query from the data plugin. This step ensures that if we refresh the page, filter and query still persists their previous values. + - Then we set up subscriptions for both filter and query, so whenever we change the values for either of them, the new state get synced up with `OsdUrlStateStorage`. + - In the return function, we unsubscribe each one of them. + +2. `connectToQueryState()` + - This function take three input parameters: query state managers, `state container`, and the four configs that it helps syncing, `filter`, `query`, `time` and `refresh intervals` + - For initial syncing, we get the initial values from the state managers in the data plugin, and we store the values in the state container + - Then we set up subscriptions for each one of the config in data plugin, so whenever we change the values for any one of them, the new state get saved and sync with the state container. + - We also set up subscriptions for the states in state container, so whenever the value in state container get changed, the state managers in the data plugin will also be updated. + - In the return function, we unsubscribe each one of them. + +There are a couple differences between the above two functions: +1. `connectStorageToQueryState()` uses `OsdUrlStateStorage` for syncing, while `connectToQueryState()` uses `state container` for syncing, and `state container` is then synced up with `OsdUrlStateStorage`. +2. `connectStorageToQueryState()` can be used for persisting the app states, specifically app filter and query values, while `connectToQueryState()` can be used for persisting both app states and global states, specificlly app filter and query which are part of app states, global filter, time range, time refresh interval which are parts of global states. +3. `connectStorageToQueryState()` sets up a one way syncing from data to `OsdUrlStateStorage`, while `connectToQueryState()` sets up two-way syncing of the data and `state container`. Both of the functions serve to connect data services to achieve state syncing. \ No newline at end of file diff --git a/src/plugins/data/public/query/state_sync/connect_to_query_state.test.ts b/src/plugins/data/public/query/state_sync/connect_to_query_state.test.ts index 5b4e0f228069..6b540c3da5c5 100644 --- a/src/plugins/data/public/query/state_sync/connect_to_query_state.test.ts +++ b/src/plugins/data/public/query/state_sync/connect_to_query_state.test.ts @@ -31,19 +31,32 @@ import { Subscription } from 'rxjs'; import { FilterManager } from '../filter_manager'; import { getFilter } from '../filter_manager/test_helpers/get_stub_filter'; -import { Filter, FilterStateStore, UI_SETTINGS } from '../../../common'; +import { Filter, FilterStateStore, Query, UI_SETTINGS } from '../../../common'; import { coreMock } from '../../../../../core/public/mocks'; import { BaseStateContainer, createStateContainer, + IOsdUrlStateStorage, + createOsdUrlStateStorage, Storage, } from '../../../../opensearch_dashboards_utils/public'; import { QueryService, QueryStart } from '../query_service'; import { StubBrowserStorage } from '../../../../../test_utils/public/stub_browser_storage'; -import { connectToQueryState } from './connect_to_query_state'; +import { connectStorageToQueryState, connectToQueryState } from './connect_to_query_state'; import { TimefilterContract } from '../timefilter'; import { QueryState } from './types'; - +import { createBrowserHistory, History } from 'history'; +import { QueryStringContract } from '../query_string'; + +const connectStorageToQueryStateFn = ( + query: QueryStart, + OsdUrlStateStorage: IOsdUrlStateStorage +) => { + connectStorageToQueryState(query, OsdUrlStateStorage, { + filters: FilterStateStore.APP_STATE, + query: true, + }); +}; const connectToQueryGlobalState = (query: QueryStart, state: BaseStateContainer) => connectToQueryState(query, state, { refreshInterval: true, @@ -74,6 +87,110 @@ setupMock.uiSettings.get.mockImplementation((key: string) => { } }); +describe('connect_storage_to_query_state', () => { + let queryServiceStart: QueryStart; + let queryString: QueryStringContract; + let queryChangeSub: Subscription; + let queryChangeTriggered = jest.fn(); + let filterManager: FilterManager; + let filterManagerChangeSub: Subscription; + let filterManagerChangeTriggered = jest.fn(); + let osdUrlStateStorage: IOsdUrlStateStorage; + let history: History; + let gF1: Filter; + let gF2: Filter; + let aF1: Filter; + let aF2: Filter; + let q1: Query; + + beforeEach(() => { + const queryService = new QueryService(); + queryService.setup({ + uiSettings: setupMock.uiSettings, + storage: new Storage(new StubBrowserStorage()), + }); + queryServiceStart = queryService.start({ + uiSettings: setupMock.uiSettings, + storage: new Storage(new StubBrowserStorage()), + savedObjectsClient: startMock.savedObjects.client, + }); + + queryString = queryServiceStart.queryString; + queryChangeTriggered = jest.fn(); + queryChangeSub = queryString.getUpdates$().subscribe(queryChangeTriggered); + + filterManager = queryServiceStart.filterManager; + filterManagerChangeTriggered = jest.fn(); + filterManagerChangeSub = filterManager.getUpdates$().subscribe(filterManagerChangeTriggered); + + window.location.href = '/'; + history = createBrowserHistory(); + osdUrlStateStorage = createOsdUrlStateStorage({ useHash: false, history }); + + gF1 = getFilter(FilterStateStore.GLOBAL_STATE, true, true, 'key1', 'value1'); + gF2 = getFilter(FilterStateStore.GLOBAL_STATE, false, false, 'key2', 'value2'); + aF1 = getFilter(FilterStateStore.APP_STATE, true, true, 'key3', 'value3'); + aF2 = getFilter(FilterStateStore.APP_STATE, false, false, 'key4', 'value4'); + + q1 = { + query: 'count is less than 100', + language: 'kuery', + }; + }); + + afterEach(() => { + filterManagerChangeSub.unsubscribe(); + queryChangeSub.unsubscribe(); + }); + + test('state is initialized with default state', () => { + expect(osdUrlStateStorage.get('_q')).toBeNull(); + connectStorageToQueryStateFn(queryServiceStart, osdUrlStateStorage); + + expect(osdUrlStateStorage.get('_q')).toEqual({ + query: queryString.getDefaultQuery(), + filters: filterManager.getAppFilters(), + }); + }); + + test('state is initialized with URL states', () => { + const initialStates = { + filters: [aF1, aF2], + query: q1, + }; + osdUrlStateStorage.set('_q', initialStates, { + replace: true, + }); + connectStorageToQueryStateFn(queryServiceStart, osdUrlStateStorage); + expect(filterManager.getFilters().length).toBe(2); + expect(queryString.getQuery()).toStrictEqual(q1); + }); + + test('when global filter changes, filter in storage should not be updated', () => { + connectStorageToQueryStateFn(queryServiceStart, osdUrlStateStorage); + const previousStorage = osdUrlStateStorage.get('_q'); + filterManager.setFilters([gF1, gF1]); + const updatedStorage = osdUrlStateStorage.get('_q'); + expect(previousStorage).toStrictEqual(updatedStorage); + }); + + test('when app filter changes, filter storage should be updated', () => { + connectStorageToQueryStateFn(queryServiceStart, osdUrlStateStorage); + const previousStorage = osdUrlStateStorage.get('_q'); + filterManager.setFilters([aF1, aF1]); + const updatedStorage = osdUrlStateStorage.get('_q'); + expect(previousStorage).not.toStrictEqual(updatedStorage); + }); + + test('when query changes, state updates query', () => { + connectStorageToQueryStateFn(queryServiceStart, osdUrlStateStorage); + const previousStorage = osdUrlStateStorage.get('_q'); + queryString.setQuery(q1); + const updatedStorage = osdUrlStateStorage.get('_q'); + expect(previousStorage).not.toStrictEqual(updatedStorage); + }); +}); + describe('connect_to_global_state', () => { let queryServiceStart: QueryStart; let filterManager: FilterManager; diff --git a/src/plugins/data/public/query/state_sync/connect_to_query_state.ts b/src/plugins/data/public/query/state_sync/connect_to_query_state.ts index 2f492fd28f75..8b850b36eabc 100644 --- a/src/plugins/data/public/query/state_sync/connect_to_query_state.ts +++ b/src/plugins/data/public/query/state_sync/connect_to_query_state.ts @@ -31,12 +31,113 @@ import { Subscription } from 'rxjs'; import { filter, map } from 'rxjs/operators'; import _ from 'lodash'; -import { BaseStateContainer } from '../../../../opensearch_dashboards_utils/public'; +import { + BaseStateContainer, + IOsdUrlStateStorage, +} from '../../../../opensearch_dashboards_utils/public'; import { QuerySetup, QueryStart } from '../query_service'; import { QueryState, QueryStateChange } from './types'; import { FilterStateStore, COMPARE_ALL_OPTIONS, compareFilters } from '../../../common'; import { validateTimeRange } from '../timefilter'; +/** + * Helper function to sync up filter and query services in data plugin + * with a URL state storage so plugins can persist the app filter and query + * values across refresh + * @param QueryService: either setup or start + * @param OsdUrlStateStorage to use for syncing and store data + * @param syncConfig app filter and query + */ +export const connectStorageToQueryState = ( + { + filterManager, + queryString, + state$, + }: Pick, + OsdUrlStateStorage: IOsdUrlStateStorage, + syncConfig: { + filters: FilterStateStore; + query: boolean; + } +) => { + try { + const syncKeys: Array = []; + if (syncConfig.query) { + syncKeys.push('query'); + } + if (syncConfig.filters === FilterStateStore.APP_STATE) { + syncKeys.push('appFilters'); + } + + const initialStateFromURL: QueryState = OsdUrlStateStorage.get('_q') ?? { + query: queryString.getDefaultQuery(), + filters: filterManager.getAppFilters(), + }; + + // set up initial '_q' flag in the URL to sync query and filter changes + if (!OsdUrlStateStorage.get('_q')) { + OsdUrlStateStorage.set('_q', initialStateFromURL, { + replace: true, + }); + } + + if (syncConfig.query && !_.isEqual(initialStateFromURL.query, queryString.getQuery())) { + if (initialStateFromURL.query) { + queryString.setQuery(_.cloneDeep(initialStateFromURL.query)); + } + } + + if (syncConfig.filters === FilterStateStore.APP_STATE) { + if ( + !initialStateFromURL.filters || + !compareFilters(initialStateFromURL.filters, filterManager.getAppFilters(), { + ...COMPARE_ALL_OPTIONS, + state: false, + }) + ) { + if (initialStateFromURL.filters) { + filterManager.setAppFilters(_.cloneDeep(initialStateFromURL.filters)); + } + } + + const subs: Subscription[] = [ + state$ + .pipe( + filter(({ changes }) => { + return syncKeys.some((syncKey) => changes[syncKey]); + }), + map(({ changes, state }) => { + const newState: QueryState = { + query: state.query, + filters: state.filters, + }; + if (syncConfig.query && changes.query) { + newState.query = queryString.getQuery(); + } + + if (syncConfig.filters === FilterStateStore.APP_STATE && changes.appFilters) { + newState.filters = filterManager.getAppFilters(); + } + + return newState; + }) + ) + .subscribe((newState) => { + OsdUrlStateStorage.set('_q', newState, { + replace: true, + }); + }), + ]; + + return () => { + subs.forEach((s) => s.unsubscribe()); + }; + } + } catch (err) { + return; + } +}; + /** * Helper to setup two-way syncing of global data and a state container * @param QueryService: either setup or start diff --git a/src/plugins/data/public/query/state_sync/index.ts b/src/plugins/data/public/query/state_sync/index.ts index 7109d7d87020..5c5619461e08 100644 --- a/src/plugins/data/public/query/state_sync/index.ts +++ b/src/plugins/data/public/query/state_sync/index.ts @@ -28,6 +28,6 @@ * under the License. */ -export { connectToQueryState } from './connect_to_query_state'; +export { connectToQueryState, connectStorageToQueryState } from './connect_to_query_state'; export { syncQueryStateWithUrl } from './sync_state_with_url'; export { QueryState, QueryStateChange } from './types'; diff --git a/src/plugins/vis_builder/public/application/components/top_nav.tsx b/src/plugins/vis_builder/public/application/components/top_nav.tsx index fe9b735b5acf..d64352fb4d69 100644 --- a/src/plugins/vis_builder/public/application/components/top_nav.tsx +++ b/src/plugins/vis_builder/public/application/components/top_nav.tsx @@ -18,6 +18,7 @@ import { setEditorState } from '../utils/state_management/metadata_slice'; import { useCanSave } from '../utils/use/use_can_save'; import { saveStateToSavedObject } from '../../saved_visualizations/transforms'; import { TopNavMenuData } from '../../../../navigation/public'; +import { opensearchFilters, connectStorageToQueryState } from '../../../../data/public'; export const TopNav = () => { // id will only be set for the edit route @@ -34,6 +35,10 @@ export const TopNav = () => { const saveDisabledReason = useCanSave(); const savedVisBuilderVis = useSavedVisBuilderVis(visualizationIdFromUrl); + connectStorageToQueryState(services.data.query, services.osdUrlStateStorage, { + filters: opensearchFilters.FilterStateStore.APP_STATE, + query: true, + }); const { selected: indexPattern } = useIndexPatterns(); const [config, setConfig] = useState(); const originatingApp = useTypedSelector((state) => {