slow return for single runQuery on multiple requests

[jsnick]

Hi

Test environment

• Node v16, @google-cloud/datastore v7.5.1

Test script

• test-script-version-a.js

import {Datastore} from '@google-cloud/datastore';
import {v4 as uuidv4} from 'uuid';

void (async () => {
  const projectId = 'my-project-id';
  const datastore = new Datastore({
    projectId,
  });
  const size = 1; // will be set in different value
  await Promise.all(Array.from(Array(size)).map(() => (async () => {
    const label = uuidv4();
    console.time(label);
    const cid = 'dm:4783269862244352:6109546586112000';
    await datastore.runQuery(
      datastore.createQuery('ChatMessage').filter('cids', cid).limit(200),
    );
    console.timeEnd(label);
  })()));
})();

• test-script-version-b.js

import {Datastore} from '@google-cloud/datastore';
import {v4 as uuidv4} from 'uuid';

void (async () => {
  const projectId = 'my-project-id';
  const size = 1; // will be set in different value
  await Promise.all(Array.from(Array(size)).map(() => (async () => {
    const datastore = new Datastore({
      projectId,
    });
    const label = uuidv4();
    console.time(label);
    const cid = 'dm:4783269862244352:6109546586112000';
    await datastore.runQuery(
      datastore.createQuery('ChatMessage').filter('cids', cid).limit(200),
    );
    console.timeEnd(label);
  })()));
})();

Result

• size: 1

$ node test-version-a.js
7fa015b5-e31f-41bd-be2c-13636dec4390: 5.684s
$ node test-version-b.js
32f879a9-1db6-4e97-9b95-25044ad92e8e: 5.665s

• size: 50

$ node test-version-a.js // size 50
a785ac4f-22a1-4e3b-8fd0-50e358ed07f8: 35.678s
e64ed1d8-a14a-4867-a9d6-a649ce46d3a0: 36.844s
ceb4e7b7-1725-49e9-a751-29975cdc1b8f: 36.868s
177b7282-db81-4804-8716-74706ac1eca0: 36.924s
ad89a9cf-9123-4a69-a865-e3c613a03fd9: 36.947s
8f63b09f-a2dc-466a-856d-a549b4d1cce4: 37.021s
b50f6710-6415-4d55-81b8-ec248a09de4c: 37.175s
f1f8fe63-775b-46e8-b416-0238dae8fe75: 37.413s
cd8fd0b5-e0ac-415a-8e7e-5ea45fec3693: 37.592s
c0d2bc90-3365-4fd4-8755-ebf5b47e3457: 37.611s
ed4dda2e-b337-4f05-bcb5-4897030d7f86: 37.827s
80b9ed5d-550d-4738-84fd-6ace1f9cdb90: 37.864s
2cd0f5b2-7bfe-4bb8-b867-c3902d11644b: 38.083s
6990e9d2-3580-426f-b227-1a05202a753b: 38.140s
cd941f53-e13b-4820-967c-a4fc69e11166: 38.295s
ec7aec7e-7b84-4a73-8a39-3751b9cf6552: 38.398s
73f02200-1426-4f8c-926e-c93de6d8e0ac: 38.437s
9debce5e-99d2-4609-a0df-ace94e70920b: 38.474s
6b50af41-0995-47dd-bf41-88b7fc05eea3: 38.533s
67845df3-380f-4934-8e9f-af07b283e625: 38.555s
0a4c0c66-796a-4e1e-b925-550e56de850d: 38.647s
603c42b3-d26a-4274-b91d-ad51025718f6: 38.648s
3d1c1354-5167-4ff6-89cd-4494069649aa: 38.691s
fff44dc7-55b0-4247-9269-e7118d1c289a: 38.728s
3b638a49-3b12-4164-b437-9f9372e17b5d: 38.770s
120235d5-b4d4-48fe-83a0-9eb4edbe1b4c: 38.836s
43a186b0-8f61-42a6-b20b-9b8ed3123400: 38.834s
4afe30e2-ff64-47b9-bd7d-abb6fdd84725: 38.939s
1affa0e9-ccf8-4f23-94ee-438ca86788cb: 38.936s
9414b7b5-ba97-4c34-b85d-db94277da470: 39.024s
1992da0a-6b43-44b7-b9cc-e0d9b18698c5: 39.077s
bf474d8f-c534-4019-9700-d2e558c83488: 39.121s
198c7c07-e953-4fac-8c91-428eb200d841: 39.199s
0a6479fb-dfb6-45ee-95d7-c378a07160e2: 39.292s
f630bc78-9637-4935-a25a-c6f679ab8167: 39.316s
6764c7d3-618f-4f89-8b69-54b1715b3925: 39.413s
f7272a65-aa5e-4292-ae34-df734cb26b87: 39.417s
c4a954d5-ba2c-4a28-b4c9-bba028793017: 39.536s
ac9a3ff0-01b8-4dd4-b4b3-a40321c86be7: 39.536s
e29e903a-0576-4b58-b816-5eb276153abc: 39.538s
6316353c-ab76-4363-b6cd-26c14e9ed3c3: 39.548s
31bef9f6-d7e8-45b7-9a65-35d009822e0d: 39.609s
afb4eb1d-0da9-463b-8b03-f51a982ef7e9: 39.612s
4b1ad464-1c85-412b-af5a-053ac8026388: 39.655s
23c2fd8a-e288-4a14-86ac-d2386e987e51: 39.755s
3097f37b-dd1a-4366-800f-dee743889e13: 39.776s
cf73912b-f75c-4d1a-84b6-90db1f2c6d65: 39.825s
2ca77571-6dea-4487-8263-0eb863dc0a3a: 39.856s
2628ccf4-066a-449b-8c67-d0ca7ade3c0c: 39.924s
a1f80ade-dd49-4d84-bafb-3d6a4a402e16: 39.923s

$ node test-version-b.js // size 50
f84490d9-b7bc-4435-adba-825fa17d3637: 16.320s
5e295967-56ee-4696-9abf-44acc9f9d7c3: 18.687s
d5c31702-96f1-49ce-bb88-47372e5cd933: 19.252s
e3fb541c-c056-439f-a102-fed5e1c75044: 19.385s
a8c2f04d-492b-4fea-bef7-34c9839515ba: 19.386s
3c11d4ca-91c1-4426-9afa-523d3418be3c: 19.390s
cce266a9-4c0d-4033-81c3-c534df670492: 19.552s
52f4e434-cd88-4d1d-9c73-e2c271083995: 19.550s
5605c947-ea65-41e3-a8d9-9b4956cf5d8f: 20.208s
af02ec99-1e36-4469-bcd7-447532ae46a9: 20.211s
591103e3-12ab-4930-a616-bcc36aef726f: 20.369s
62d31c1f-d7f1-4ee0-91a9-6631accd95ce: 20.404s
efc79ef2-3c62-40ac-aae7-5e97e8b2b966: 20.495s
1878d342-1462-4de2-863e-38781c168a67: 20.574s
03514b82-3ea6-4121-ae62-f07e158bbf4c: 20.573s
a830cb95-7c13-4e05-a617-06d0a7d61224: 20.946s
f0141846-e6af-4c4e-b6fc-cf626caf656d: 21.056s
30f6d075-c133-4af7-9edb-4a6987e0eda8: 21.168s
7ae207bf-bfad-4014-a55b-f97f6753ed18: 21.236s
bca0aa9d-3db4-4c7c-88e3-4c5288ff009e: 21.308s
62a2d2fc-4628-4008-be07-0726cd2ad867: 21.344s
2a3f8881-0d73-4417-99ff-70747274efd5: 21.420s
67cda6d0-b6e2-439c-b338-d5a1f0de752c: 21.575s
98c2327b-d35f-4f91-9bc6-856feb193af6: 21.573s
aaee03f0-bebc-4d1b-a3a3-455138839d48: 21.577s
11b88046-c0e4-4667-878d-d3f0c9bdf694: 21.640s
59f9023a-801e-46a6-a6fc-8aa9166de172: 21.700s
634915e0-052d-4deb-b73d-7715c799591b: 21.702s
cd3981fd-ccaf-417c-935c-1bec62e0c3ca: 22.318s
06657821-88ac-4189-a08f-722717425f46: 22.439s
5b2f7044-f5c9-4777-8b01-14cbe33d1063: 22.524s
0317c405-03d9-40f7-a7c2-2ca6405613f2: 22.526s
0f7849b6-bb35-4b3a-a754-656d9dbbc6fa: 22.524s
0d823bf7-fa7b-4efe-83e1-fb830593f547: 22.524s
540c0e2a-29f6-4295-b820-d3768567bea5: 22.569s
d94d7201-5784-4edd-99bf-63ac184831a8: 22.568s
0da2746a-ff29-4c5c-a1a5-7adaa5a18977: 22.596s
37b6b28b-44fa-403c-b6df-b27dcdbeb57a: 22.638s
875b2a93-0d5d-4be4-bf0d-7ac0cf4b9146: 22.778s
9107c411-5a1b-4af2-9eb0-69bcf7c7ae55: 22.777s
0f0a9aa6-2a5a-4a07-b66d-19d4ec1ce841: 22.774s
9e72b75b-d8f8-447f-af8d-011d297f3933: 22.775s
01834925-ed72-4ea3-a597-458632183866: 22.775s
73405a2d-9406-4bc1-b7c3-d11ac75d531e: 22.800s
300c6735-6bb7-44ba-a753-c162b66574b8: 22.855s
6761544c-97e7-468b-ac08-6c5516c19f5f: 22.859s
d1ea4eac-2ae9-4051-8dbf-8cdcc390d4cd: 22.901s
deec4fc9-9ab0-4c8e-9f36-b2c357b74d02: 22.901s
379372cd-89d3-41cc-a62a-ec90337844a5: 22.949s
56f46f94-d023-4df7-b368-67d226c2fed0: 25.160s

I don't know why these are happend.
Are these expected result?

Long latency like version-a result If there are many requests ,which use a little long query on datastore, on one of the instances (app engine standard node env)

I tried changing server code like version-b (new Datastore every requests), but I met OOM T^T.

any suggestions ?

[jsnick]

ah, maybe because client pool (like nodejs-firestore) doesn't exist in nodejs-datastore ?

[danieljbruce]

Logs from system tests in https://github.com/googleapis/nodejs-datastore/pull/1158/files show that latency is not an issue anymore so closing this issue.