update import api only

src/core/server/saved_objects/import/check_conflict_for_data_source.test.ts

@@ -0,0 +1,182 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Any modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { mockUuidv4 } from './__mocks__';
+import { SavedObjectReference, SavedObjectsImportRetry } from 'opensearch-dashboards/public';
+import { SavedObject } from '../types';
+import { SavedObjectsErrorHelpers } from '..';
+import {
+  checkConflictsForDataSource,
+  ConflictsForDataSourceParams,
+} from './check_conflict_for_data_source';
+
+type SavedObjectType = SavedObject<{ title?: string }>;
+
+/**
+ * Function to create a realistic-looking import object given a type and ID
+ */
+const createObject = (type: string, id: string): SavedObjectType => ({
+  type,
+  id,
+  attributes: { title: 'some-title' },
+  references: (Symbol() as unknown) as SavedObjectReference[],
+});
+
+const getResultMock = {
+  conflict: (type: string, id: string) => {
+    const error = SavedObjectsErrorHelpers.createConflictError(type, id).output.payload;
+    return { type, id, error };
+  },
+  unresolvableConflict: (type: string, id: string) => {
+    const conflictMock = getResultMock.conflict(type, id);
+    const metadata = { isNotOverwritable: true };
+    return { ...conflictMock, error: { ...conflictMock.error, metadata } };
+  },
+  invalidType: (type: string, id: string) => {
+    const error = SavedObjectsErrorHelpers.createUnsupportedTypeError(type).output.payload;
+    return { type, id, error };
+  },
+};
+
+/**
+ * Create a variety of different objects to exercise different import / result scenarios
+ */
+const obj1 = createObject('type-1', 'id-1'); // -> success
+const obj2 = createObject('type-2', 'id-2'); // -> conflict
+const obj3 = createObject('type-3', 'id-3'); // -> unresolvable conflict
+const obj4 = createObject('type-4', 'id-4'); // -> invalid type
+const dsObj = createObject('data-source', 'data-source-id-1'); // -> data-source type, no need to add in the filteredObjects
+const objects = [obj1, obj2, obj3, obj4];
+const dsObj1 = createObject('type-1', 'ds_id-1'); // -> object with data source id
+const dsObj2 = createObject('type-2', 'ds_id-2'); // -> object with data source id
+const objectsWithDataSource = [dsObj, dsObj1, dsObj2];
+const dsObj1Error = getResultMock.conflict(dsObj1.type, dsObj1.id);
+
+describe('#checkConflictsForDataSource', () => {
+  const setupParams = (partial: {
+    objects: SavedObjectType[];
+    ignoreRegularConflicts?: boolean;
+    retries?: SavedObjectsImportRetry[];
+    createNewCopies?: boolean;
+    dataSourceId?: string;
+  }): ConflictsForDataSourceParams => {
+    return { ...partial };
+  };
+
+  beforeEach(() => {
+    mockUuidv4.mockReset();
+    mockUuidv4.mockReturnValueOnce(`new-object-id`);
+  });
+
+  it('exits early if there are no objects to check', async () => {
+    const params = setupParams({ objects: [] });
+    const checkConflictsForDataSourceResult = await checkConflictsForDataSource(params);
+    expect(checkConflictsForDataSourceResult).toEqual({
+      filteredObjects: [],
+      errors: [],
+      importIdMap: new Map(),
+      pendingOverwrites: new Set(),
+    });
+  });
+
+  it('returns original objects result when there is no data source id', async () => {
+    const params = setupParams({ objects });
+    const checkConflictsForDataSourceResult = await checkConflictsForDataSource(params);
+    expect(checkConflictsForDataSourceResult).toEqual({
+      filteredObjects: [...objects],
+      errors: [],
+      importIdMap: new Map(),
+      pendingOverwrites: new Set(),
+    });
+  });
+
+  it('return obj if it is not data source obj and there is no conflict of the data source id', async () => {
+    const params = setupParams({ objects: objectsWithDataSource, dataSourceId: 'ds' });
+    const checkConflictsForDataSourceResult = await checkConflictsForDataSource(params);
+    expect(checkConflictsForDataSourceResult).toEqual({
+      filteredObjects: [dsObj1, dsObj2],
+      errors: [],
+      importIdMap: new Map(),
+      pendingOverwrites: new Set(),
+    });
+  });
+
+  it('can resolve the data source id conflict when the ds it not match when ignoreRegularConflicts=true', async () => {
+    const params = setupParams({
+      objects: objectsWithDataSource,
+      ignoreRegularConflicts: true,
+      dataSourceId: 'currentDsId',
+    });
+    const checkConflictsForDataSourceResult = await checkConflictsForDataSource(params);
+
+    expect(checkConflictsForDataSourceResult).toEqual(
+      expect.objectContaining({
+        filteredObjects: [
+          {
+            ...dsObj1,
+            id: 'currentDsId_id-1',
+          },
+          {
+            ...dsObj2,
+            id: 'currentDsId_id-2',
+          },
+        ],
+        errors: [],
+        importIdMap: new Map([
+          [`${dsObj1.type}:${dsObj1.id}`, { id: 'currentDsId_id-1', omitOriginId: true }],
+          [`${dsObj2.type}:${dsObj2.id}`, { id: 'currentDsId_id-2', omitOriginId: true }],
+        ]),
+        pendingOverwrites: new Set([`${dsObj1.type}:${dsObj1.id}`, `${dsObj2.type}:${dsObj2.id}`]),
+      })
+    );
+  });
+
+  it('can push error when do not override with data source conflict', async () => {
+    const params = setupParams({
+      objects: [dsObj1],
+      ignoreRegularConflicts: false,
+      dataSourceId: 'currentDs',
+    });
+    const checkConflictsForDataSourceResult = await checkConflictsForDataSource(params);
+    expect(checkConflictsForDataSourceResult).toEqual({
+      filteredObjects: [],
+      errors: [
+        {
+          ...dsObj1Error,
+          title: dsObj1.attributes.title,
+          meta: { title: dsObj1.attributes.title },
+          error: { type: 'conflict' },
+        },
+      ],
+      importIdMap: new Map(),
+      pendingOverwrites: new Set(),
+    });
+  });
+});

src/core/server/saved_objects/import/check_conflict_for_data_source.ts

@@ -0,0 +1,115 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Any modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { SavedObject, SavedObjectsImportError, SavedObjectsImportRetry } from '../types';
+
+export interface ConflictsForDataSourceParams {
+  objects: Array<SavedObject<{ title?: string }>>;
+  ignoreRegularConflicts?: boolean;
+  retries?: SavedObjectsImportRetry[];
+  createNewCopies?: boolean;
+  dataSourceId?: string;
+}
+
+/**
+ * function to check the conflict when enabled multiple data source
+ * the purpose of this function is to check the conflict of the imported saved objects and data source
+ * @param objects, this the array of saved objects to be verified whether contains the data source conflict
+ * @param ignoreRegularConflicts whether to override
+ * @param dataSourceId the id to identify the data source
+ * @returns {filteredObjects, errors, importIdMap, pendingOverwrites }
+ */
+export async function checkConflictsForDataSource({
+  objects,
+  ignoreRegularConflicts,
+  retries = [],
+  dataSourceId,
+}: ConflictsForDataSourceParams) {
+  const filteredObjects: Array<SavedObject<{ title?: string }>> = [];
+  const errors: SavedObjectsImportError[] = [];
+  const importIdMap = new Map<string, { id?: string; omitOriginId?: boolean }>();
+  const pendingOverwrites = new Set<string>();
+
+  // exit early if there are no objects to check
+  if (objects.length === 0) {
+    return { filteredObjects, errors, importIdMap, pendingOverwrites };
+  }
+  const retryMap = retries.reduce(
+    (acc, cur) => acc.set(`${cur.type}:${cur.id}`, cur),
+    new Map<string, SavedObjectsImportRetry>()
+  );
+  objects.forEach((object) => {
+    const {
+      type,
+      id,
+      attributes: { title },
+    } = object;
+    const { destinationId } = retryMap.get(`${type}:${id}`) || {};
+
+    const currentDataSourceId = dataSourceId;
+
+    if (!object.type.includes('data-source')) {
+      // check the previous data source existed or not
+      // by extract it from the id
+      // e.g. e0c9e490-bdd7-11ee-b216-d78a57002330_ff959d40-b880-11e8-a6d9-e546fe2bba5f
+      // e0c9e490-bdd7-11ee-b216-d78a57002330 is the data source id
+      // for saved object data source itself, e0c9e490-bdd7-11ee-b216-d78a57002330 return undefined
+      const parts = id.split('_'); // this is the array to host the split results of the id
+      const previoudDataSourceId = parts.length > 1 ? parts[0] : undefined;
+      // case for import saved object from osd exported
+      // when the imported daved objects with the different dataSourceId comparing to the current dataSourceId
+      // previous data source id not exist, push it to filtered object
+      // no conflict
+      if (!previoudDataSourceId || previoudDataSourceId === currentDataSourceId) {
+        filteredObjects.push(object);
+      } else if (previoudDataSourceId && previoudDataSourceId !== currentDataSourceId) {
+        if (ignoreRegularConflicts) {
+          // overwrite
+          // ues old key and new value in the importIdMap
+          // old key is used to look up, new key is used to be the id of new object
+          const omitOriginId = ignoreRegularConflicts;
+          // e.g. e0c9e490-bdd7-11ee-b216-d78a57002330_ff959d40-b880-11e8-a6d9-e546fe2bba5f
+          // rawId is ff959d40-b880-11e8-a6d9-e546fe2bba5f
+          const rawId = parts[1];
+          importIdMap.set(`${type}:${id}`, { id: `${currentDataSourceId}_${rawId}`, omitOriginId });
+          pendingOverwrites.add(`${type}:${id}`);
+          filteredObjects.push({ ...object, id: `${currentDataSourceId}_${rawId}` });
+        } else {
+          // not override
+          // push error
+          const error = { type: 'conflict' as 'conflict', ...(destinationId && { destinationId }) };
+          errors.push({ type, id, title, meta: { title }, error });
+        }
+      }
+    }
+  });
+
+  return { filteredObjects, errors, importIdMap, pendingOverwrites };
+}

src/core/server/saved_objects/import/check_conflicts.ts

@@ -44,6 +44,7 @@ interface CheckConflictsParams {
   ignoreRegularConflicts?: boolean;
   retries?: SavedObjectsImportRetry[];
   createNewCopies?: boolean;
+  dataSourceId?: string;
 }
 
 const isUnresolvableConflict = (error: SavedObjectError) =>
@@ -66,7 +67,6 @@ export async function checkConflicts({
   if (objects.length === 0) {
     return { filteredObjects, errors, importIdMap, pendingOverwrites };
   }
-
   const retryMap = retries.reduce(
     (acc, cur) => acc.set(`${cur.type}:${cur.id}`, cur),
     new Map<string, SavedObjectsImportRetry>()

src/core/server/saved_objects/import/check_origin_conflicts.ts

@@ -45,6 +45,7 @@ interface CheckOriginConflictsParams {
   namespace?: string;
   ignoreRegularConflicts?: boolean;
   importIdMap: Map<string, unknown>;
+  dataSourceId?: string;
 }
 
 type CheckOriginConflictParams = Omit<CheckOriginConflictsParams, 'objects'> & {
@@ -186,6 +187,7 @@ export async function checkOriginConflicts({ objects, ...params }: CheckOriginCo
     if (sources.length === 1 && destinations.length === 1) {
       // This is a simple "inexact match" result -- a single import object has a single destination conflict.
       if (params.ignoreRegularConflicts) {
+        // importIdMap.set(`${type}:${id}`, { id: dataSourceId ? `${dataSourceId}_${destinations[0].id}` : `${destinations[0].id}` });
         importIdMap.set(`${type}:${id}`, { id: destinations[0].id });
         pendingOverwrites.add(`${type}:${id}`);
       } else {

src/core/server/saved_objects/import/collect_saved_objects.ts

@@ -46,13 +46,15 @@ interface CollectSavedObjectsOptions {
   objectLimit: number;
   filter?: (obj: SavedObject) => boolean;
   supportedTypes: string[];
+  dataSourceId?: string;
 }
 
 export async function collectSavedObjects({
   readStream,
   objectLimit,
   filter,
   supportedTypes,
+  dataSourceId,
 }: CollectSavedObjectsOptions) {
   const errors: SavedObjectsImportError[] = [];
   const entries: Array<{ type: string; id: string }> = [];
@@ -79,7 +81,12 @@ export async function collectSavedObjects({
     }),
     createFilterStream<SavedObject>((obj) => (filter ? filter(obj) : true)),
     createMapStream((obj: SavedObject) => {
-      importIdMap.set(`${obj.type}:${obj.id}`, {});
+      if (dataSourceId) {
+        importIdMap.set(`${dataSourceId}_${obj.type}:${obj.id}`, {});
+      } else {
+        importIdMap.set(`${obj.type}:${obj.id}`, {});
+      }
+
       // Ensure migrations execute on every saved object
       return Object.assign({ migrationVersion: {} }, obj);
     }),