From a44b09fd9f81ca1a10909a81b3a553a5c37e1617 Mon Sep 17 00:00:00 2001 From: Manasvini B Suryanarayana Date: Tue, 27 Jun 2023 12:53:05 -0700 Subject: [PATCH] Migrate from legacy elasticsearch client to opensearch-js client in `osd-opensearch-archiver` package (#4142) Signed-off-by: Manasvini B Suryanarayana Signed-off-by: Josh Romero Co-authored-by: Josh Romero --- CHANGELOG.md | 1 + packages/osd-opensearch-archiver/package.json | 6 +- .../empty_opensearch_dashboards_index.ts | 2 +- .../src/actions/load.ts | 4 +- .../src/actions/save.ts | 2 +- .../src/actions/unload.ts | 2 +- packages/osd-opensearch-archiver/src/cli.ts | 13 ++-- .../docs/generate_doc_records_stream.test.ts | 28 ++++--- .../lib/docs/generate_doc_records_stream.ts | 14 ++-- .../lib/docs/index_doc_records_stream.test.ts | 77 +++++++++++-------- .../src/lib/docs/index_doc_records_stream.ts | 6 +- .../src/lib/docs/test_stubs.ts | 2 +- .../lib/indices/create_index_stream.test.ts | 1 - .../src/lib/indices/create_index_stream.ts | 2 +- .../src/lib/indices/delete_index.ts | 16 ++-- .../src/lib/indices/delete_index_stream.ts | 2 +- .../generate_index_records_stream.test.ts | 4 +- .../indices/generate_index_records_stream.ts | 7 +- .../indices/opensearch_dashboards_index.ts | 34 ++++---- .../src/lib/indices/test_stubs.ts | 9 +-- .../src/opensearch_archiver.ts | 2 +- test/common/services/opensearch_archiver.ts | 2 +- 22 files changed, 130 insertions(+), 106 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b1df894e994f..b24377e61b1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fix EUI/OUI type errors ([#3798](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3798)) - Remove unused Sass in `tile_map` plugin ([#4110](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/4110)) - [Table Visualization] Remove custom styling for text-align:center in favor of OUI utility class. ([#4164](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/4164)) +- Migrate from legacy elasticsearch client to opensearch-js client in `osd-opensearch-archiver` package([#4142](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/4142)) - Replace the use of `bluebird` in `saved_objects` plugin ([#4026](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/4026)) - [Vis Colors] Replace color maps with OUI color palettes ([#4293](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/4293)) - [Vis Colors] [Maps] Replace hardcoded color to OUI color in `maps_legacy` plugin ([#4294](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/4294)) diff --git a/packages/osd-opensearch-archiver/package.json b/packages/osd-opensearch-archiver/package.json index 8f88fa879525..1c036dc10c50 100644 --- a/packages/osd-opensearch-archiver/package.json +++ b/packages/osd-opensearch-archiver/package.json @@ -12,9 +12,7 @@ }, "dependencies": { "@osd/dev-utils": "1.0.0", - "elasticsearch": "^16.7.0" + "@opensearch-project/opensearch": "^2.2.0" }, - "devDependencies": { - "@types/elasticsearch": "^5.0.33" - } + "devDependencies": {} } diff --git a/packages/osd-opensearch-archiver/src/actions/empty_opensearch_dashboards_index.ts b/packages/osd-opensearch-archiver/src/actions/empty_opensearch_dashboards_index.ts index e0b23ac8088f..2f7ba1f22fd0 100644 --- a/packages/osd-opensearch-archiver/src/actions/empty_opensearch_dashboards_index.ts +++ b/packages/osd-opensearch-archiver/src/actions/empty_opensearch_dashboards_index.ts @@ -28,7 +28,7 @@ * under the License. */ -import { Client } from 'elasticsearch'; +import { Client } from '@opensearch-project/opensearch'; import { ToolingLog, OsdClient } from '@osd/dev-utils'; import { diff --git a/packages/osd-opensearch-archiver/src/actions/load.ts b/packages/osd-opensearch-archiver/src/actions/load.ts index f513d049dbd1..65297501264a 100644 --- a/packages/osd-opensearch-archiver/src/actions/load.ts +++ b/packages/osd-opensearch-archiver/src/actions/load.ts @@ -32,7 +32,7 @@ import { resolve } from 'path'; import { createReadStream } from 'fs'; import { Readable } from 'stream'; import { ToolingLog, OsdClient } from '@osd/dev-utils'; -import { Client } from 'elasticsearch'; +import { Client } from '@opensearch-project/opensearch'; import { createPromiseFromStreams, concatStreamProviders } from '../lib/streams'; @@ -114,7 +114,7 @@ export async function loadAction({ await client.indices.refresh({ index: '_all', - allowNoIndices: true, + allow_no_indices: true, }); // If we affected the OpenSearch Dashboards index, we need to ensure it's migrated... diff --git a/packages/osd-opensearch-archiver/src/actions/save.ts b/packages/osd-opensearch-archiver/src/actions/save.ts index 46f2103f8f51..5e5b0d310d5c 100644 --- a/packages/osd-opensearch-archiver/src/actions/save.ts +++ b/packages/osd-opensearch-archiver/src/actions/save.ts @@ -31,7 +31,7 @@ import { resolve } from 'path'; import { createWriteStream, mkdirSync } from 'fs'; import { Readable, Writable } from 'stream'; -import { Client } from 'elasticsearch'; +import { Client } from '@opensearch-project/opensearch'; import { ToolingLog } from '@osd/dev-utils'; import { createListStream, createPromiseFromStreams } from '../lib/streams'; diff --git a/packages/osd-opensearch-archiver/src/actions/unload.ts b/packages/osd-opensearch-archiver/src/actions/unload.ts index 3e2bb4c8535d..6896b7878c6f 100644 --- a/packages/osd-opensearch-archiver/src/actions/unload.ts +++ b/packages/osd-opensearch-archiver/src/actions/unload.ts @@ -31,7 +31,7 @@ import { resolve } from 'path'; import { createReadStream } from 'fs'; import { Readable, Writable } from 'stream'; -import { Client } from 'elasticsearch'; +import { Client } from '@opensearch-project/opensearch'; import { ToolingLog, OsdClient } from '@osd/dev-utils'; import { createPromiseFromStreams } from '../lib/streams'; diff --git a/packages/osd-opensearch-archiver/src/cli.ts b/packages/osd-opensearch-archiver/src/cli.ts index 002e3e287587..e8606ed26645 100644 --- a/packages/osd-opensearch-archiver/src/cli.ts +++ b/packages/osd-opensearch-archiver/src/cli.ts @@ -38,8 +38,8 @@ import Path from 'path'; import readline from 'readline'; import { RunWithCommands, createFlagError } from '@osd/dev-utils'; +import { Client, ClientOptions } from '@opensearch-project/opensearch'; import { readConfigFile } from '@osd/test'; -import legacyElasticsearch from 'elasticsearch'; import { OpenSearchArchiver } from './opensearch_archiver'; @@ -56,7 +56,7 @@ export function runCli() { default: ${defaultConfigPath} --opensearch-url url for OpenSearch, prefer the --config flag --opensearch-dashboards-url url for OpenSearch Dashboards, prefer the --config flag - --dir where arechives are stored, prefer the --config flag + --dir where archives are stored, prefer the --config flag `, }, async extendContext({ log, flags, addCleanupTask }) { @@ -99,10 +99,11 @@ export function runCli() { throw createFlagError('--dir or --config must be defined'); } - const client = new legacyElasticsearch.Client({ - host: opensearchUrl, - log: flags.verbose ? 'trace' : [], - }); + const clientOptions: ClientOptions = { + node: opensearchUrl.toString(), + }; + + const client = new Client(clientOptions); addCleanupTask(() => client.close()); const opensearchArchiver = new OpenSearchArchiver({ diff --git a/packages/osd-opensearch-archiver/src/lib/docs/generate_doc_records_stream.test.ts b/packages/osd-opensearch-archiver/src/lib/docs/generate_doc_records_stream.test.ts index 462aba6c234a..603bfc278e2d 100644 --- a/packages/osd-opensearch-archiver/src/lib/docs/generate_doc_records_stream.test.ts +++ b/packages/osd-opensearch-archiver/src/lib/docs/generate_doc_records_stream.test.ts @@ -47,9 +47,11 @@ describe('opensearchArchiver: createGenerateDocRecordsStream()', () => { expect(params).to.have.property('index', 'logstash-*'); expect(params).to.have.property('size', 1000); return { - hits: { - total: 0, - hits: [], + body: { + hits: { + total: 0, + hits: [], + }, }, }; }, @@ -74,9 +76,11 @@ describe('opensearchArchiver: createGenerateDocRecordsStream()', () => { expect(params).to.have.property('scroll', '1m'); expect(params).to.have.property('rest_total_hits_as_int', true); return { - hits: { - total: 0, - hits: [], + body: { + hits: { + total: 0, + hits: [], + }, }, }; }, @@ -101,17 +105,19 @@ describe('opensearchArchiver: createGenerateDocRecordsStream()', () => { expect(params).to.have.property('index', 'index1'); await delay(200); return { - _scroll_id: 'index1ScrollId', - hits: { total: 2, hits: [{ _id: 1, _index: '.opensearch_dashboards_1' }] }, + body: { + _scroll_id: 'index1ScrollId', + hits: { total: 2, hits: [{ _id: 1, _index: '.opensearch_dashboards_1' }] }, + }, }; }, async (name, params) => { expect(name).to.be('scroll'); - expect(params).to.have.property('scrollId', 'index1ScrollId'); + expect(params).to.have.property('scroll_id', 'index1ScrollId'); expect(Date.now() - checkpoint).to.not.be.lessThan(200); checkpoint = Date.now(); await delay(200); - return { hits: { total: 2, hits: [{ _id: 2, _index: 'foo' }] } }; + return { body: { hits: { total: 2, hits: [{ _id: 2, _index: 'foo' }] } } }; }, async (name, params) => { expect(name).to.be('search'); @@ -119,7 +125,7 @@ describe('opensearchArchiver: createGenerateDocRecordsStream()', () => { expect(Date.now() - checkpoint).to.not.be.lessThan(200); checkpoint = Date.now(); await delay(200); - return { hits: { total: 0, hits: [] } }; + return { body: { hits: { total: 0, hits: [] } } }; }, ]); diff --git a/packages/osd-opensearch-archiver/src/lib/docs/generate_doc_records_stream.ts b/packages/osd-opensearch-archiver/src/lib/docs/generate_doc_records_stream.ts index 6949e2b4d4ff..9bbb99898600 100644 --- a/packages/osd-opensearch-archiver/src/lib/docs/generate_doc_records_stream.ts +++ b/packages/osd-opensearch-archiver/src/lib/docs/generate_doc_records_stream.ts @@ -29,7 +29,7 @@ */ import { Transform } from 'stream'; -import { Client, SearchParams, SearchResponse } from 'elasticsearch'; +import { Client, ApiResponse } from '@opensearch-project/opensearch'; import { Stats } from '../stats'; import { Progress } from '../progress'; @@ -53,7 +53,7 @@ export function createGenerateDocRecordsStream({ async transform(index, enc, callback) { try { let remainingHits = 0; - let resp: SearchResponse | null = null; + let resp: ApiResponse | null = null; while (!resp || remainingHits > 0) { if (!resp) { @@ -66,17 +66,17 @@ export function createGenerateDocRecordsStream({ query, }, rest_total_hits_as_int: true, // not declared on SearchParams type - } as SearchParams); - remainingHits = resp.hits.total; + }); + remainingHits = resp.body.hits.total; progress.addToTotal(remainingHits); } else { resp = await client.scroll({ - scrollId: resp._scroll_id!, + scroll_id: resp.body._scroll_id!, scroll: SCROLL_TIMEOUT, }); } - for (const hit of resp.hits.hits) { + for (const hit of resp.body?.hits.hits) { remainingHits -= 1; stats.archivedDoc(hit._index); this.push({ @@ -94,7 +94,7 @@ export function createGenerateDocRecordsStream({ }); } - progress.addToComplete(resp.hits.hits.length); + progress.addToComplete(resp.body.hits.hits.length); } callback(undefined); diff --git a/packages/osd-opensearch-archiver/src/lib/docs/index_doc_records_stream.test.ts b/packages/osd-opensearch-archiver/src/lib/docs/index_doc_records_stream.test.ts index 00b9e22660ca..313fdf3589ed 100644 --- a/packages/osd-opensearch-archiver/src/lib/docs/index_doc_records_stream.test.ts +++ b/packages/osd-opensearch-archiver/src/lib/docs/index_doc_records_stream.test.ts @@ -63,11 +63,12 @@ describe('opensearchArchiver: createIndexDocRecordsStream()', () => { const client = createStubClient([ async (name, params) => { expect(name).to.be('bulk'); - expect(params).to.eql({ - body: recordsToBulkBody(records), - requestTimeout: 120000, - }); - return { ok: true }; + expect(params).to.eql({ body: recordsToBulkBody(records) }); + return { + body: { + ok: true, + }, + }; }, ]); const stats = createStubStats(); @@ -88,19 +89,21 @@ describe('opensearchArchiver: createIndexDocRecordsStream()', () => { const client = createStubClient([ async (name, params) => { expect(name).to.be('bulk'); - expect(params).to.eql({ - body: recordsToBulkBody(records.slice(0, 1)), - requestTimeout: 120000, - }); - return { ok: true }; + expect(params).to.eql({ body: recordsToBulkBody(records.slice(0, 1)) }); + return { + body: { + ok: true, + }, + }; }, async (name, params) => { expect(name).to.be('bulk'); - expect(params).to.eql({ - body: recordsToBulkBody(records.slice(1)), - requestTimeout: 120000, - }); - return { ok: true }; + expect(params).to.eql({ body: recordsToBulkBody(records.slice(1)) }); + return { + body: { + ok: true, + }, + }; }, ]); const stats = createStubStats(); @@ -124,21 +127,23 @@ describe('opensearchArchiver: createIndexDocRecordsStream()', () => { const client = createStubClient([ async (name, params) => { expect(name).to.be('bulk'); - expect(params).to.eql({ - body: recordsToBulkBody(records.slice(0, 1)), - requestTimeout: 120000, - }); + expect(params).to.eql({ body: recordsToBulkBody(records.slice(0, 1)) }); await delay(delayMs); - return { ok: true }; + return { + body: { + ok: true, + }, + }; }, async (name, params) => { expect(name).to.be('bulk'); - expect(params).to.eql({ - body: recordsToBulkBody(records.slice(1)), - requestTimeout: 120000, - }); + expect(params).to.eql({ body: recordsToBulkBody(records.slice(1)) }); expect(Date.now() - start).to.not.be.lessThan(delayMs); - return { ok: true }; + return { + body: { + ok: true, + }, + }; }, ]); const progress = new Progress(); @@ -160,17 +165,29 @@ describe('opensearchArchiver: createIndexDocRecordsStream()', () => { async (name, params) => { expect(name).to.be('bulk'); expect(params.body.length).to.eql(1 * 2); - return { ok: true }; + return { + body: { + ok: true, + }, + }; }, async (name, params) => { expect(name).to.be('bulk'); expect(params.body.length).to.eql(299 * 2); - return { ok: true }; + return { + body: { + ok: true, + }, + }; }, async (name, params) => { expect(name).to.be('bulk'); expect(params.body.length).to.eql(1 * 2); - return { ok: true }; + return { + body: { + ok: true, + }, + }; }, ]); const progress = new Progress(); @@ -189,8 +206,8 @@ describe('opensearchArchiver: createIndexDocRecordsStream()', () => { const records = createPersonDocRecords(2); const stats = createStubStats(); const client = createStubClient([ - async () => ({ ok: true }), - async () => ({ errors: true, forcedError: true }), + async () => ({ body: { ok: true } }), + async () => ({ body: { errors: true, forcedError: true } }), ]); const progress = new Progress(); diff --git a/packages/osd-opensearch-archiver/src/lib/docs/index_doc_records_stream.ts b/packages/osd-opensearch-archiver/src/lib/docs/index_doc_records_stream.ts index d1ba68f7bf05..4fe7059b8464 100644 --- a/packages/osd-opensearch-archiver/src/lib/docs/index_doc_records_stream.ts +++ b/packages/osd-opensearch-archiver/src/lib/docs/index_doc_records_stream.ts @@ -28,7 +28,7 @@ * under the License. */ -import { Client } from 'elasticsearch'; +import { Client } from '@opensearch-project/opensearch'; import { Writable } from 'stream'; import { Stats } from '../stats'; import { Progress } from '../progress'; @@ -58,8 +58,8 @@ export function createIndexDocRecordsStream( ); }); - const resp = await client.bulk({ requestTimeout: 2 * 60 * 1000, body }); - if (resp.errors) { + const resp = await client.bulk({ body }, { requestTimeout: 2 * 60 * 1000 }); + if (resp.body.errors) { throw new Error(`Failed to index all documents: ${JSON.stringify(resp, null, 2)}`); } } diff --git a/packages/osd-opensearch-archiver/src/lib/docs/test_stubs.ts b/packages/osd-opensearch-archiver/src/lib/docs/test_stubs.ts index 27b5803cc31b..5630f95ba143 100644 --- a/packages/osd-opensearch-archiver/src/lib/docs/test_stubs.ts +++ b/packages/osd-opensearch-archiver/src/lib/docs/test_stubs.ts @@ -28,7 +28,7 @@ * under the License. */ -import { Client } from 'elasticsearch'; +import { Client } from '@opensearch-project/opensearch'; import sinon from 'sinon'; import Chance from 'chance'; import { times } from 'lodash'; diff --git a/packages/osd-opensearch-archiver/src/lib/indices/create_index_stream.test.ts b/packages/osd-opensearch-archiver/src/lib/indices/create_index_stream.test.ts index 200a6a79f965..b78f8aa51411 100644 --- a/packages/osd-opensearch-archiver/src/lib/indices/create_index_stream.test.ts +++ b/packages/osd-opensearch-archiver/src/lib/indices/create_index_stream.test.ts @@ -83,7 +83,6 @@ describe('opensearchArchiver: createCreateIndexStream()', () => { expect((client.indices.getAlias as sinon.SinonSpy).calledOnce).to.be.ok(); expect((client.indices.getAlias as sinon.SinonSpy).args[0][0]).to.eql({ name: 'existing-index', - ignore: [404], }); expect((client.indices.delete as sinon.SinonSpy).calledOnce).to.be.ok(); expect((client.indices.delete as sinon.SinonSpy).args[0][0]).to.eql({ diff --git a/packages/osd-opensearch-archiver/src/lib/indices/create_index_stream.ts b/packages/osd-opensearch-archiver/src/lib/indices/create_index_stream.ts index 6154f0382a96..7427f8e7ae21 100644 --- a/packages/osd-opensearch-archiver/src/lib/indices/create_index_stream.ts +++ b/packages/osd-opensearch-archiver/src/lib/indices/create_index_stream.ts @@ -31,7 +31,7 @@ import { Transform, Readable } from 'stream'; import { inspect } from 'util'; -import { Client } from 'elasticsearch'; +import { Client } from '@opensearch-project/opensearch'; import { ToolingLog } from '@osd/dev-utils'; import { Stats } from '../stats'; diff --git a/packages/osd-opensearch-archiver/src/lib/indices/delete_index.ts b/packages/osd-opensearch-archiver/src/lib/indices/delete_index.ts index ee8fe10d8d0b..93c8f4fdd039 100644 --- a/packages/osd-opensearch-archiver/src/lib/indices/delete_index.ts +++ b/packages/osd-opensearch-archiver/src/lib/indices/delete_index.ts @@ -29,7 +29,7 @@ */ import { get } from 'lodash'; -import { Client } from 'elasticsearch'; +import { Client } from '@opensearch-project/opensearch'; import { ToolingLog } from '@osd/dev-utils'; import { Stats } from '../stats'; @@ -46,8 +46,8 @@ export async function deleteIndex(options: { const { client, stats, index, log, retryIfSnapshottingCount = 10 } = options; const getIndicesToDelete = async () => { - const aliasInfo = await client.indices.getAlias({ name: index, ignore: [404] }); - return aliasInfo.status === 404 ? [index] : Object.keys(aliasInfo); + const { body, statusCode } = await client.indices.getAlias({ name: index }, { ignore: [404] }); + return statusCode === 404 ? [index] : Object.keys(body); }; try { @@ -92,7 +92,9 @@ export function isDeleteWhileSnapshotInProgressError(error: object) { export async function waitForSnapshotCompletion(client: Client, index: string, log: ToolingLog) { const isSnapshotPending = async (repository: string, snapshot: string) => { const { - snapshots: [status], + body: { + snapshots: [status], + }, } = await client.snapshot.status({ repository, snapshot, @@ -103,7 +105,9 @@ export async function waitForSnapshotCompletion(client: Client, index: string, l }; const getInProgressSnapshots = async (repository: string) => { - const { snapshots: inProgressSnapshots } = await client.snapshot.get({ + const { + body: { snapshots: inProgressSnapshots }, + } = await client.snapshot.get({ repository, snapshot: '_current', }); @@ -112,7 +116,7 @@ export async function waitForSnapshotCompletion(client: Client, index: string, l for (const repository of Object.keys(await client.snapshot.getRepository({} as any))) { const allInProgress = await getInProgressSnapshots(repository); - const found = allInProgress.find((s: any) => s.indices.includes(index)); + const found = allInProgress?.find((s: any) => s.indices.includes(index)); if (!found) { continue; diff --git a/packages/osd-opensearch-archiver/src/lib/indices/delete_index_stream.ts b/packages/osd-opensearch-archiver/src/lib/indices/delete_index_stream.ts index b728aa35757b..fe22f31e15da 100644 --- a/packages/osd-opensearch-archiver/src/lib/indices/delete_index_stream.ts +++ b/packages/osd-opensearch-archiver/src/lib/indices/delete_index_stream.ts @@ -29,7 +29,7 @@ */ import { Transform } from 'stream'; -import { Client } from 'elasticsearch'; +import { Client } from '@opensearch-project/opensearch'; import { ToolingLog } from '@osd/dev-utils'; import { Stats } from '../stats'; diff --git a/packages/osd-opensearch-archiver/src/lib/indices/generate_index_records_stream.test.ts b/packages/osd-opensearch-archiver/src/lib/indices/generate_index_records_stream.test.ts index a961cca3550e..8d0b4087ea40 100644 --- a/packages/osd-opensearch-archiver/src/lib/indices/generate_index_records_stream.test.ts +++ b/packages/osd-opensearch-archiver/src/lib/indices/generate_index_records_stream.test.ts @@ -67,8 +67,8 @@ describe('opensearchArchiver: createGenerateIndexRecordsStream()', () => { ]); const params = (client.indices.get as sinon.SinonSpy).args[0][0]; - expect(params).to.have.property('filterPath'); - const filters: string[] = params.filterPath; + expect(params).to.have.property('filter_path'); + const filters: string[] = params.filter_path; expect(filters.some((path) => path.includes('index.creation_date'))).to.be(true); expect(filters.some((path) => path.includes('index.uuid'))).to.be(true); expect(filters.some((path) => path.includes('index.version'))).to.be(true); diff --git a/packages/osd-opensearch-archiver/src/lib/indices/generate_index_records_stream.ts b/packages/osd-opensearch-archiver/src/lib/indices/generate_index_records_stream.ts index f682af3d8598..e74965657c70 100644 --- a/packages/osd-opensearch-archiver/src/lib/indices/generate_index_records_stream.ts +++ b/packages/osd-opensearch-archiver/src/lib/indices/generate_index_records_stream.ts @@ -29,7 +29,7 @@ */ import { Transform } from 'stream'; -import { Client } from 'elasticsearch'; +import { Client } from '@opensearch-project/opensearch'; import { Stats } from '../stats'; export function createGenerateIndexRecordsStream(client: Client, stats: Stats) { @@ -40,7 +40,7 @@ export function createGenerateIndexRecordsStream(client: Client, stats: Stats) { try { const resp = (await client.indices.get({ index: indexOrAlias, - filterPath: [ + filter_path: [ '*.settings', '*.mappings', // remove settings that aren't really settings @@ -56,9 +56,10 @@ export function createGenerateIndexRecordsStream(client: Client, stats: Stats) { })) as Record; for (const [index, { settings, mappings }] of Object.entries(resp)) { + const { body, statusCode } = await client.indices.getAlias({ index }); const { [index]: { aliases }, - } = await client.indices.getAlias({ index }); + } = statusCode === 404 ? {} : body; stats.archivedIndex(index, { settings, mappings }); this.push({ diff --git a/packages/osd-opensearch-archiver/src/lib/indices/opensearch_dashboards_index.ts b/packages/osd-opensearch-archiver/src/lib/indices/opensearch_dashboards_index.ts index aff69cd2682c..9a415ebb4ee4 100644 --- a/packages/osd-opensearch-archiver/src/lib/indices/opensearch_dashboards_index.ts +++ b/packages/osd-opensearch-archiver/src/lib/indices/opensearch_dashboards_index.ts @@ -28,7 +28,7 @@ * under the License. */ -import { Client, CreateDocumentParams } from 'elasticsearch'; +import { Client } from '@opensearch-project/opensearch'; import { ToolingLog, OsdClient } from '@osd/dev-utils'; import { Stats } from '../stats'; import { deleteIndex } from './delete_index'; @@ -52,15 +52,17 @@ export async function deleteOpenSearchDashboardsIndices({ await client.indices.putSettings({ index: indexNames, - body: { index: { blocks: { read_only: false } } }, + body: { blocks: { read_only: false } }, }); - await deleteIndex({ - client, - stats, - index: indexNames, - log, - }); + for (const indexName of indexNames) { + await deleteIndex({ + client, + stats, + index: indexName, + log, + }); + } return indexNames; } @@ -101,8 +103,8 @@ async function fetchOpenSearchDashboardsIndices(client: Client) { format: 'json', }); const isOpenSearchDashboardsIndex = (index: string) => /^\.kibana(:?_\d*)?$/.test(index); - return opensearchDashboardsIndices - .map((x: { index: string }) => x.index) + return opensearchDashboardsIndices.body + .map((x) => x.index?.toString() ?? '') .filter(isOpenSearchDashboardsIndex); } @@ -133,21 +135,19 @@ export async function cleanOpenSearchDashboardsIndices({ bool: { must_not: { ids: { - type: '_doc', values: ['space:default'], }, }, }, }, }, - ignore: [409], }); - if (resp.total !== resp.deleted) { + if (resp.body.total !== resp.body.deleted) { log.warning( 'delete by query deleted %d of %d total documents, trying again', - resp.deleted, - resp.total + resp.body.deleted, + resp.body.total ); continue; } @@ -166,9 +166,7 @@ export async function cleanOpenSearchDashboardsIndices({ export async function createDefaultSpace({ index, client }: { index: string; client: Client }) { await client.create({ index, - type: '_doc', id: 'space:default', - ignore: 409, body: { type: 'space', updated_at: new Date().toISOString(), @@ -179,5 +177,5 @@ export async function createDefaultSpace({ index, client }: { index: string; cli _reserved: true, }, }, - } as CreateDocumentParams); + }); } diff --git a/packages/osd-opensearch-archiver/src/lib/indices/test_stubs.ts b/packages/osd-opensearch-archiver/src/lib/indices/test_stubs.ts index 64b42a629cd6..bab84dd653b2 100644 --- a/packages/osd-opensearch-archiver/src/lib/indices/test_stubs.ts +++ b/packages/osd-opensearch-archiver/src/lib/indices/test_stubs.ts @@ -28,7 +28,7 @@ * under the License. */ -import { Client } from 'elasticsearch'; +import { Client } from '@opensearch-project/opensearch'; import sinon from 'sinon'; import { ToolingLog } from '@osd/dev-utils'; import { Stats } from '../stats'; @@ -113,14 +113,13 @@ export const createStubClient = ( getAlias: sinon.spy(async ({ index, name }) => { if (index && existingIndices.indexOf(index) >= 0) { const result = indexAlias(aliases, index); - return { [index]: { aliases: result ? { [result]: {} } : {} } }; + return { body: { [index]: { aliases: result ? { [result]: {} } : {} } } }; } if (name && aliases[name]) { - return { [aliases[name]]: { aliases: { [name]: {} } } }; + return { body: { [aliases[name]]: { aliases: { [name]: {} } } } }; } - - return { status: 404 }; + return { statusCode: 404 }; }), updateAliases: sinon.spy(async ({ body }) => { body.actions.forEach( diff --git a/packages/osd-opensearch-archiver/src/opensearch_archiver.ts b/packages/osd-opensearch-archiver/src/opensearch_archiver.ts index 9e6a1246b184..bb831b05c0b0 100644 --- a/packages/osd-opensearch-archiver/src/opensearch_archiver.ts +++ b/packages/osd-opensearch-archiver/src/opensearch_archiver.ts @@ -28,7 +28,7 @@ * under the License. */ -import { Client } from 'elasticsearch'; +import { Client } from '@opensearch-project/opensearch'; import { ToolingLog, OsdClient } from '@osd/dev-utils'; import { diff --git a/test/common/services/opensearch_archiver.ts b/test/common/services/opensearch_archiver.ts index 3985c7aaf466..b422ef69068f 100644 --- a/test/common/services/opensearch_archiver.ts +++ b/test/common/services/opensearch_archiver.ts @@ -39,7 +39,7 @@ export function OpenSearchArchiverProvider({ hasService, }: FtrProviderContext): OpenSearchArchiver { const config = getService('config'); - const client = getService('legacyOpenSearch'); + const client = getService('opensearch'); const log = getService('log'); if (!config.get('opensearchArchiver')) {