Skip to content

Commit a2ebbf4

Browse files
shortcutsshortcuts
authored
feat(javascript): add replaceAllObjectsWithTransformation (#5008)
1 parent 2dfc215 commit a2ebbf4

File tree

9 files changed

+580
-13
lines changed

9 files changed

+580
-13
lines changed

‎playground/javascript/node/algoliasearch.ts

Lines changed: 18 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -136,18 +136,24 @@ async function testAlgoliasearchBridgeIngestion() {
136136
// Init client with appId and apiKey
137137
const client = algoliasearch(appId, adminApiKey, { transformation: { region: 'eu' } });
138138

139-
await client.saveObjectsWithTransformation({
140-
indexName: 'foo',
141-
objects: [{ objectID: 'foo', data: { baz: 'baz', win: 42 } }],
142-
waitForTasks: true,
143-
});
144-
145-
await client.partialUpdateObjectsWithTransformation({
146-
indexName: 'foo',
147-
objects: [{ objectID: 'foo', data: { baz: 'baz', win: 42 } }],
148-
waitForTasks: true,
149-
createIfNotExists: false,
150-
});
139+
// console.log('saveObjectsWithTransformation', await client.saveObjectsWithTransformation({
140+
// indexName: 'foo',
141+
// objects: [{ objectID: 'foo', data: { baz: 'baz', win: 42 } }],
142+
// waitForTasks: true,
143+
// }));
144+
//
145+
// console.log('partialUpdateObjectsWithTransformation', await client.partialUpdateObjectsWithTransformation({
146+
// indexName: 'foo',
147+
// objects: [{ objectID: 'foo', data: { baz: 'baz', win: 42 } }],
148+
// waitForTasks: true,
149+
// createIfNotExists: false,
150+
// }));
151+
152+
console.log('replaceAllObjectsWithTransformation', await client.replaceAllObjectsWithTransformation({
153+
indexName: 'boyd',
154+
objects: [{ objectID: 'foo', data: { baz: 'baz', win: 42 } }, { objectID: 'bar', data: { baz: 'baz', win: 24 } }],
155+
batchSize: 2
156+
}));
151157
}
152158

153159
// testAlgoliasearch();

‎scripts/cts/runCts.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import { assertPushMockValid } from './testServer/pushMock.ts';
1313
import { assertValidReplaceAllObjects } from './testServer/replaceAllObjects.ts';
1414
import { assertValidReplaceAllObjectsFailed } from './testServer/replaceAllObjectsFailed.ts';
1515
import { assertValidReplaceAllObjectsScopes } from './testServer/replaceAllObjectsScopes.ts';
16+
import { assertValidReplaceAllObjectsWithTransformation } from './testServer/replaceAllObjectsWithTransformation.ts';
1617
import { assertValidTimeouts } from './testServer/timeout.ts';
1718
import { assertValidWaitForApiKey } from './testServer/waitFor.ts';
1819

@@ -154,6 +155,7 @@ export async function runCts(
154155
assertValidTimeouts(languages.length);
155156
assertChunkWrapperValid(languages.length - skip('dart'));
156157
assertValidReplaceAllObjects(languages.length - skip('dart'));
158+
assertValidReplaceAllObjectsWithTransformation(only('javascript'));
157159
assertValidAccountCopyIndex(only('javascript'));
158160
assertValidReplaceAllObjectsFailed(languages.length - skip('dart'));
159161
assertValidReplaceAllObjectsScopes(languages.length - skip('dart'));

‎scripts/cts/testServer/index.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import { pushMockServer } from './pushMock.ts';
1717
import { replaceAllObjectsServer } from './replaceAllObjects.ts';
1818
import { replaceAllObjectsServerFailed } from './replaceAllObjectsFailed.ts';
1919
import { replaceAllObjectsScopesServer } from './replaceAllObjectsScopes.ts';
20+
import { replaceAllObjectsWithTransformationServer } from './replaceAllObjectsWithTransformation.ts';
2021
import { timeoutServer } from './timeout.ts';
2122
import { timeoutServerBis } from './timeoutBis.ts';
2223
import { waitForApiKeyServer } from './waitFor.ts';
@@ -37,6 +38,7 @@ export async function startTestServer(suites: Record<CTSType, boolean>): Promise
3738
apiKeyServer(),
3839
algoliaMockServer(),
3940
pushMockServer(),
41+
replaceAllObjectsWithTransformationServer(),
4042
);
4143
}
4244
if (suites.benchmark) {

‎scripts/cts/testServer/replaceAllObjectsWithTransformation.ts

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
import type { Server } from 'http';
2+
3+
import { expect } from 'chai';
4+
import type { Express } from 'express';
5+
import express from 'express';
6+
7+
import { setupServer } from './index.ts';
8+
9+
const raowtState: Record<
10+
string,
11+
{
12+
copyCount: number;
13+
pushCount: number;
14+
tmpIndexName: string;
15+
waitTaskCount: number;
16+
waitingForFinalWaitTask: boolean;
17+
successful: boolean;
18+
}
19+
> = {};
20+
21+
export function assertValidReplaceAllObjectsWithTransformation(expectedCount: number): void {
22+
expect(Object.keys(raowtState)).to.have.length(expectedCount);
23+
for (const lang in raowtState) {
24+
expect(raowtState[lang].successful).to.equal(true);
25+
}
26+
}
27+
28+
function addRoutes(app: Express): void {
29+
app.use(express.urlencoded({ extended: true }));
30+
app.use(
31+
express.json({
32+
type: ['application/json', 'text/plain'], // the js client sends the body as text/plain
33+
}),
34+
);
35+
36+
app.post('/1/indexes/:indexName/operation', (req, res) => {
37+
expect(req.params.indexName).to.match(/^cts_e2e_replace_all_objects_with_transformation_(.*)$/);
38+
39+
switch (req.body.operation) {
40+
case 'copy': {
41+
expect(req.params.indexName).to.not.include('tmp');
42+
expect(req.body.destination).to.include('tmp');
43+
expect(req.body.scope).to.deep.equal(['settings', 'rules', 'synonyms']);
44+
45+
const lang = req.params.indexName.replace('cts_e2e_replace_all_objects_with_transformation_', '');
46+
if (!raowtState[lang] || raowtState[lang].successful) {
47+
raowtState[lang] = {
48+
copyCount: 1,
49+
pushCount: 0,
50+
waitTaskCount: 0,
51+
tmpIndexName: req.body.destination,
52+
waitingForFinalWaitTask: false,
53+
successful: false,
54+
};
55+
} else {
56+
raowtState[lang].copyCount++;
57+
}
58+
59+
res.json({ taskID: 123 + raowtState[lang].copyCount, updatedAt: '2021-01-01T00:00:00.000Z' });
60+
break;
61+
}
62+
case 'move': {
63+
const lang = req.body.destination.replace('cts_e2e_replace_all_objects_with_transformation_', '');
64+
expect(raowtState).to.include.keys(lang);
65+
expect(raowtState[lang]).to.deep.equal({
66+
copyCount: 2,
67+
pushCount: 10,
68+
waitTaskCount: 2,
69+
tmpIndexName: req.params.indexName,
70+
waitingForFinalWaitTask: false,
71+
successful: false,
72+
});
73+
74+
expect(req.body.scope).to.equal(undefined);
75+
76+
raowtState[lang].waitingForFinalWaitTask = true;
77+
78+
res.json({ taskID: 777, updatedAt: '2021-01-01T00:00:00.000Z' });
79+
80+
break;
81+
}
82+
default:
83+
res.status(400).json({
84+
message: `invalid operation: ${req.body.operation}, body: ${JSON.stringify(req.body)}`,
85+
});
86+
}
87+
});
88+
89+
app.post('/1/push/:indexName', (req, res) => {
90+
const lang = req.params.indexName.match(
91+
/^cts_e2e_replace_all_objects_with_transformation_(.*)_tmp_\d+$/,
92+
)?.[1] as string;
93+
expect(raowtState).to.include.keys(lang);
94+
expect(req.body.action === 'addObject').to.equal(true);
95+
96+
raowtState[lang].pushCount += req.body.records.length;
97+
98+
res.json({
99+
runID: 'b1b7a982-524c-40d2-bb7f-48aab075abda',
100+
eventID: `113b2068-6337-4c85-b5c2-e7b213d8292${raowtState[lang].pushCount}`,
101+
message: 'OK',
102+
createdAt: '2022-05-12T06:24:30.049Z',
103+
});
104+
});
105+
106+
app.get('/1/runs/:runID/events/:eventID', (req, res) => {
107+
res.json({ status: 'finished' });
108+
});
109+
110+
app.get('/1/indexes/:indexName/task/:taskID', (req, res) => {
111+
const lang = req.params.indexName.match(
112+
/^cts_e2e_replace_all_objects_with_transformation_(.*)_tmp_\d+$/,
113+
)?.[1] as string;
114+
expect(raowtState).to.include.keys(lang);
115+
116+
raowtState[lang].waitTaskCount++;
117+
if (raowtState[lang].waitingForFinalWaitTask) {
118+
expect(req.params.taskID).to.equal('777');
119+
expect(raowtState[lang].waitTaskCount).to.equal(3);
120+
121+
raowtState[lang].successful = true;
122+
}
123+
124+
res.json({ status: 'published', updatedAt: '2021-01-01T00:00:00.000Z' });
125+
});
126+
}
127+
128+
export function replaceAllObjectsWithTransformationServer(): Promise<Server> {
129+
// this server is used to simulate the responses for the replaceAllObjectsWithTransformationServer method,
130+
// and uses a state machine to determine if the logic is correct.
131+
return setupServer('replaceAllObjectsWithTransformationServer', 6690, addRoutes);
132+
}

‎specs/search/helpers/chunkedPush.yml

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
method:
2+
get:
3+
x-helper: true
4+
tags:
5+
- Records
6+
x-available-languages:
7+
- javascript
8+
operationId: chunkedPush
9+
summary: Replace all records in an index
10+
description: |
11+
Helper: Chunks the given `objects` list in subset of 1000 elements max in order to make it fit in `push` requests by leveraging the Transformation pipeline setup in the Push connector (https://www.algolia.com/doc/guides/sending-and-managing-data/send-and-update-your-data/connectors/push/).
12+
parameters:
13+
- in: query
14+
name: indexName
15+
description: The `indexName` to replace `objects` in.
16+
required: true
17+
schema:
18+
type: string
19+
- in: query
20+
name: objects
21+
description: List of objects to replace the current objects with.
22+
required: true
23+
schema:
24+
type: array
25+
items:
26+
type: object
27+
- in: query
28+
name: action
29+
description: The `batch` `action` to perform on the given array of `objects`, defaults to `addObject`.
30+
required: false
31+
schema:
32+
$ref: '../../common/schemas/Batch.yml#/action'
33+
- in: query
34+
name: waitForTasks
35+
description: Whether or not we should wait until every `batch` tasks has been processed, this operation may slow the total execution time of this method but is more reliable.
36+
required: false
37+
schema:
38+
type: boolean
39+
- in: query
40+
name: batchSize
41+
description: The size of the chunk of `objects`. The number of `batch` calls will be equal to `length(objects) / batchSize`. Defaults to 1000.
42+
required: false
43+
schema:
44+
type: integer
45+
responses:
46+
'200':
47+
description: OK
48+
content:
49+
application/json:
50+
schema:
51+
type: array
52+
items:
53+
$ref: '../../common/schemas/ingestion/WatchResponse.yml'
54+
'400':
55+
$ref: '../../common/responses/IndexNotFound.yml'

‎specs/search/helpers/replaceAllObjectsWithTransformation.yml

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
method:
2+
get:
3+
x-helper: true
4+
tags:
5+
- Records
6+
x-available-languages:
7+
- javascript
8+
operationId: replaceAllObjectsWithTransformation
9+
summary: Replace all records in an index
10+
description: |
11+
Replace all records from your index with a new set of records by leveraging the Transformation pipeline setup in the Push connector (https://www.algolia.com/doc/guides/sending-and-managing-data/send-and-update-your-data/connectors/push/).
12+
13+
This method lets you replace all records in your index without downtime. It performs these operations:
14+
1. Copy settings, synonyms, and rules from your original index to a temporary index.
15+
2. Add your new records to the temporary index.
16+
3. Replace your original index with the temporary index.
17+
18+
Use the safe parameter to ensure that these (asynchronous) operations are performed in sequence.
19+
If there's an error duing one of these steps, the temporary index won't be deleted.
20+
This operation is rate-limited.
21+
This method creates a temporary index: your record count is temporarily doubled. Algolia doesn't count the three days with the highest number of records towards your monthly usage.
22+
If you're on a legacy plan (before July 2020), this method counts two operations towards your usage (in addition to the number of records): copySettings and moveIndex.
23+
The API key you use for this operation must have access to the index YourIndex and the temporary index YourIndex_tmp.
24+
parameters:
25+
- in: query
26+
name: indexName
27+
description: The `indexName` to replace `objects` in.
28+
required: true
29+
schema:
30+
type: string
31+
- in: query
32+
name: objects
33+
description: List of objects to replace the current objects with.
34+
required: true
35+
schema:
36+
type: array
37+
items:
38+
type: object
39+
- in: query
40+
name: batchSize
41+
description: The size of the chunk of `objects`. The number of `batch` calls will be equal to `length(objects) / batchSize`. Defaults to 1000.
42+
required: false
43+
schema:
44+
type: integer
45+
default: 1000
46+
- in: query
47+
name: scopes
48+
description: List of scopes to kepp in the index. Defaults to `settings`, `synonyms`, and `rules`.
49+
required: false
50+
schema:
51+
type: array
52+
items:
53+
$ref: '../common/enums.yml#/scopeType'
54+
responses:
55+
'200':
56+
description: OK
57+
content:
58+
application/json:
59+
schema:
60+
$ref: '#/replaceAllObjectsWithTransformationResponse'
61+
'400':
62+
$ref: '../../common/responses/IndexNotFound.yml'
63+
64+
replaceAllObjectsWithTransformationResponse:
65+
type: object
66+
additionalProperties: false
67+
properties:
68+
copyOperationResponse:
69+
description: The response of the `operationIndex` request for the `copy` operation.
70+
$ref: '../../common/responses/common.yml#/updatedAtResponse'
71+
watchResponses:
72+
type: array
73+
description: The response of the `push` request(s).
74+
items:
75+
$ref: '../../common/schemas/ingestion/WatchResponse.yml'
76+
moveOperationResponse:
77+
description: The response of the `operationIndex` request for the `move` operation.
78+
$ref: '../../common/responses/common.yml#/updatedAtResponse'
79+
required:
80+
- copyOperationResponse
81+
- watchResponses
82+
- moveOperationResponse

‎specs/search/spec.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -378,6 +378,9 @@ paths:
378378
/replaceAllObjects:
379379
$ref: 'helpers/replaceAllObjects.yml#/method'
380380

381+
/replaceAllObjectsWithTransformation:
382+
$ref: 'helpers/replaceAllObjectsWithTransformation.yml#/method'
383+
381384
/chunkedBatch:
382385
$ref: 'helpers/chunkedBatch.yml#/method'
383386

0 commit comments

Comments
 (0)