Skip to content

Commit

Permalink
chore(NA): updates from lmdb-store to lmdb (elastic#145891)
Browse files Browse the repository at this point in the history 
This PR upgrades from `lbmd-store` into `lmdb` which is the new package
and fully compatible with node `v18`.
So far my tests shows the new implementation is compatible with our
usages and I'm actually seeing a great performance boost when comparing
with main specially on subsequent calls of the same command.

This can be tested by running the following 2 times on main vs this
branch `time node scripts/kibana --config
src/cli/serve/integration_tests/__fixtures__/invalid_config.yml
--migrations.skip=true --verbose`

I verify the following on my machine

**main:**
_1st run:_ 36s
_2nd run:_ 34s

**this branch:**
_1st run:_ 35s
_2nd run:_ 6s

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
(cherry picked from commit e65c652)

# Conflicts:
#	package.json
#	packages/kbn-optimizer/BUILD.bazel
#	packages/kbn-optimizer/src/node/cache.ts
#	packages/kbn-optimizer/src/node/node_auto_tranpilation.ts
#	yarn.lock
  • Loading branch information
@mistic
mistic committed Jan 18, 2023
1 parent 2b9e617 commit e7de50f
Show file tree
Hide file tree
Showing 5 changed files with 99 additions and 45 deletions.
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -757,7 +757,7 @@
"jsondiffpatch": "0.4.1",
"license-checker": "^16.0.0",
"listr": "^0.14.1",
"lmdb-store": "^1.6.11",
"lmdb": "^2.6.9",
"marge": "^1.0.1",
"micromatch": "3.1.10",
"minimist": "^1.2.6",
Expand Down
4 changes: 2 additions & 2 deletions packages/kbn-optimizer/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ RUNTIME_DEPS = [
"@npm//execa",
"@npm//jest-diff",
"@npm//json-stable-stringify",
"@npm//lmdb-store",
"@npm//lmdb",
"@npm//loader-utils",
"@npm//node-sass",
"@npm//normalize-path",
Expand Down Expand Up @@ -75,7 +75,7 @@ TYPES_DEPS = [
"@npm//del",
"@npm//execa",
"@npm//jest-diff",
"@npm//lmdb-store",
"@npm//lmdb",
"@npm//pirates",
"@npm//resize-observer-polyfill",
"@npm//rxjs",
Expand Down
25 changes: 13 additions & 12 deletions packages/kbn-optimizer/src/node/cache.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import Path from 'path';
import { Writable } from 'stream';

import chalk from 'chalk';
import * as LmdbStore from 'lmdb-store';
import * as LmdbStore from 'lmdb';

const GLOBAL_ATIME = `${Date.now()}`;
const MINUTE = 1000 * 60;
Expand Down Expand Up @@ -44,6 +44,10 @@ export class Cache {
name: 'codes',
encoding: 'string',
maxReaders: 500,
overlappingSync: false,
noSync: true,
noMetaSync: true,
noMemInit: true,
});

// TODO: redundant 'name' syntax is necessary because of a bug that I have yet to fix
Expand All @@ -66,8 +70,8 @@ export class Cache {
// keys which haven't been used in 30 days. We use `unref()` to
// make sure this timer doesn't hold other processes open
// unexpectedly
this.timer = setTimeout(() => {
this.pruneOldKeys();
this.timer = setTimeout(async () => {
await this.pruneOldKeys();
}, 30 * MINUTE);

// timer.unref is not defined in jest which emulates the dom by default
Expand Down Expand Up @@ -104,12 +108,10 @@ export class Cache {
async update(path: string, file: { mtime: string; code: string; map: any }) {
const key = this.getKey(path);

await Promise.all([
this.safePut(this.atimes, key, GLOBAL_ATIME),
this.safePut(this.mtimes, key, file.mtime),
this.safePut(this.codes, key, file.code),
this.safePut(this.sourceMaps, key, JSON.stringify(file.map)),
]);
this.safePut(this.atimes, key, GLOBAL_ATIME);
this.safePut(this.mtimes, key, file.mtime);
this.safePut(this.codes, key, file.code);
this.safePut(this.sourceMaps, key, JSON.stringify(file.map));
}

close() {
Expand All @@ -135,9 +137,9 @@ export class Cache {
}
}

private async safePut<V>(db: LmdbStore.Database<V, string>, key: string, value: V) {
private safePut<V>(db: LmdbStore.Database<V, string>, key: string, value: V) {
try {
await db.put(key, value);
db.putSync(key, value);
this.debug('PUT', db, key);
} catch (error) {
this.logError('PUT', db, key, error);
Expand Down Expand Up @@ -167,7 +169,6 @@ export class Cache {
const validKeys: string[] = [];
const invalidKeys: string[] = [];

// @ts-expect-error See https://github.com/DoctorEvidence/lmdb-store/pull/18
for (const { key, value } of this.atimes.getRange()) {
const atime = parseInt(`${value}`, 10);
if (Number.isNaN(atime) || atime < ATIME_LIMIT) {
Expand Down
1 change: 1 addition & 0 deletions packages/kbn-optimizer/src/node/integration_tests/cache.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ it('returns undefined until values are set', async () => {
PUT [sourceMaps] prefix:bar.js
HIT [mtimes] prefix:bar.js
HIT [codes] prefix:bar.js
PUT [atimes] prefix:bar.js
HIT [sourceMaps] prefix:bar.js
"
`);
Expand Down
112 changes: 82 additions & 30 deletions yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2881,6 +2881,36 @@
resolved "https://registry.yarnpkg.com/@kwsites/promise-deferred/-/promise-deferred-1.1.1.tgz#8ace5259254426ccef57f3175bc64ed7095ed919"
integrity sha512-GaHYm+c0O9MjZRu0ongGBRbinu8gVAMd2UZjji6jVmqKtZluZnptXGWhz1E8j8D2HJ3f/yMxKAUC0b+57wncIw==

"@lmdb/lmdb-darwin-arm64@2.6.9":
version "2.6.9"
resolved "https://registry.yarnpkg.com/@lmdb/lmdb-darwin-arm64/-/lmdb-darwin-arm64-2.6.9.tgz#4b84bb0ad71e78472332920c9cf8603ea3dad0bc"
integrity sha512-QxyheKfTP9k5ZVAiddCqGUtp2AD3/BMgYfski96iIbFH0skPFO+MYARMGZuemTgyM9uieT+5oKj4FjigWplpWg==

"@lmdb/lmdb-darwin-x64@2.6.9":
version "2.6.9"
resolved "https://registry.yarnpkg.com/@lmdb/lmdb-darwin-x64/-/lmdb-darwin-x64-2.6.9.tgz#28b191a9f7a1f30462d8d179cd05598fa66ebbfc"
integrity sha512-zJ1oUepZMaqiujvWeWJRG5VHXBS3opJnjAzbd4vTVsQFT0t5rbPhHgAJ2ruR9rVrb2V1BINJZuwpjhIOg9fLCQ==

"@lmdb/lmdb-linux-arm64@2.6.9":
version "2.6.9"
resolved "https://registry.yarnpkg.com/@lmdb/lmdb-linux-arm64/-/lmdb-linux-arm64-2.6.9.tgz#274dfe11209a70c059cb55c72026c24903dde3e1"
integrity sha512-KZRet8POwKowbYZqrRqdYJ+B6l+7cWG18vMCe2sgOSuE41sEMpfRQ1mKcolt3fsr0KVbuP63aPG+dwi0wGpX9w==

"@lmdb/lmdb-linux-arm@2.6.9":
version "2.6.9"
resolved "https://registry.yarnpkg.com/@lmdb/lmdb-linux-arm/-/lmdb-linux-arm-2.6.9.tgz#7bacd104067e7dbb1bb67c907c1bc642e2d2ac96"
integrity sha512-Umw+ikxbsYZHHqr8eMycmApj6IIZCK4k1rp5/pqqx9FvAaPv4/Y63owiMLoKfipjel0YPaNyvSeXAJK3l/8Pbw==

"@lmdb/lmdb-linux-x64@2.6.9":
version "2.6.9"
resolved "https://registry.yarnpkg.com/@lmdb/lmdb-linux-x64/-/lmdb-linux-x64-2.6.9.tgz#d37b25c9b553c5d5e66055a64d118e3fd42557d9"
integrity sha512-11xFQ4kCIPGnYULcfkW4SIMIY1sukA4DHez62DKvYn+tLr4AB1o9jm1Jk6bisKFh5Cql+JUr7klHxeIuxvGZdg==

"@lmdb/lmdb-win32-x64@2.6.9":
version "2.6.9"
resolved "https://registry.yarnpkg.com/@lmdb/lmdb-win32-x64/-/lmdb-win32-x64-2.6.9.tgz#bf8e647dabd8b672744315f5df3e363b5987a463"
integrity sha512-qECZ+1j3PSarYeCmJlYlrxq3TB7S020ICrYmpxyQyphbRiMI9I1Bw4t+vPrMAEKsTqB8UaOzBp21YWUpsiCBfA==

"@loaders.gl/core@2.3.1", "@loaders.gl/core@^2.3.1":
version "2.3.1"
resolved "https://registry.yarnpkg.com/@loaders.gl/core/-/core-2.3.1.tgz#147037e17b014528dce00187aac0ec6ccb05938b"
Expand Down Expand Up @@ -18294,17 +18324,23 @@ listr@^0.14.1:
p-map "^2.0.0"
rxjs "^6.3.3"

lmdb-store@^1.6.11:
version "1.6.11"
resolved "https://registry.yarnpkg.com/lmdb-store/-/lmdb-store-1.6.11.tgz#801da597af8c7a01c81f87d5cc7a7497e381236d"
integrity sha512-hIvoGmHGsFhb2VRCmfhodA/837ULtJBwRHSHKIzhMB7WtPH6BRLPsvXp1MwD3avqGzuZfMyZDUp3tccLvr721Q==
dependencies:
nan "^2.14.2"
node-gyp-build "^4.2.3"
ordered-binary "^1.0.0"
weak-lru-cache "^1.0.0"
lmdb@^2.6.9:
version "2.6.9"
resolved "https://registry.yarnpkg.com/lmdb/-/lmdb-2.6.9.tgz#aa782ec873bcf70333b251eede9e711819ef5765"
integrity sha512-rVA3OchNoKxoD2rYhtc9nooqbJmId+vvfPzTWhanRPhdVr0hbgnF9uB9ZEHFU2lEeYVdh83Pt2H6DudeWuz+JA==
dependencies:
msgpackr "1.7.2"
node-addon-api "^4.3.0"
node-gyp-build-optional-packages "5.0.3"
ordered-binary "^1.4.0"
weak-lru-cache "^1.2.2"
optionalDependencies:
msgpackr "^1.4.7"
"@lmdb/lmdb-darwin-arm64" "2.6.9"
"@lmdb/lmdb-darwin-x64" "2.6.9"
"@lmdb/lmdb-linux-arm" "2.6.9"
"@lmdb/lmdb-linux-arm64" "2.6.9"
"@lmdb/lmdb-linux-x64" "2.6.9"
"@lmdb/lmdb-win32-x64" "2.6.9"

load-json-file@^1.0.0:
version "1.1.0"
Expand Down Expand Up @@ -19785,20 +19821,26 @@ ms@2.1.3, ms@^2.1.3:
resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2"
integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==

msgpackr-extract@^1.0.14:
version "1.0.14"
resolved "https://registry.yarnpkg.com/msgpackr-extract/-/msgpackr-extract-1.0.14.tgz#87d3fe825d226e7f3d9fe136375091137f958561"
integrity sha512-t8neMf53jNZRF+f0H9VvEUVvtjGZ21odSBRmFfjZiyxr9lKYY0mpY3kSWZAIc7YWXtCZGOvDQVx2oqcgGiRBrw==
msgpackr-extract@^2.1.2:
version "2.2.0"
resolved "https://registry.yarnpkg.com/msgpackr-extract/-/msgpackr-extract-2.2.0.tgz#4bb749b58d9764cfdc0d91c7977a007b08e8f262"
integrity sha512-0YcvWSv7ZOGl9Od6Y5iJ3XnPww8O7WLcpYMDwX+PAA/uXLDtyw94PJv9GLQV/nnp3cWlDhMoyKZIQLrx33sWog==
dependencies:
nan "^2.14.2"
node-gyp-build "^4.2.3"

msgpackr@^1.4.7:
version "1.4.7"
resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.4.7.tgz#d802ade841e7d2e873000b491cdda6574a3d5748"
integrity sha512-bhC8Ed1au3L3oHaR/fe4lk4w7PLGFcWQ5XY/Tk9N6tzDRz8YndjCG68TD8zcvYZoxNtw767eF/7VpaTpU9kf9w==
node-gyp-build-optional-packages "5.0.3"
optionalDependencies:
msgpackr-extract "^1.0.14"
"@msgpackr-extract/msgpackr-extract-darwin-arm64" "2.2.0"
"@msgpackr-extract/msgpackr-extract-darwin-x64" "2.2.0"
"@msgpackr-extract/msgpackr-extract-linux-arm" "2.2.0"
"@msgpackr-extract/msgpackr-extract-linux-arm64" "2.2.0"
"@msgpackr-extract/msgpackr-extract-linux-x64" "2.2.0"
"@msgpackr-extract/msgpackr-extract-win32-x64" "2.2.0"

msgpackr@1.7.2:
version "1.7.2"
resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.7.2.tgz#68d6debf5999d6b61abb6e7046a689991ebf7261"
integrity sha512-mWScyHTtG6TjivXX9vfIy2nBtRupaiAj0HQ2mtmpmYujAmqZmaaEVPaSZ1NKLMvicaMLFzEaMk0ManxMRg8rMQ==
optionalDependencies:
msgpackr-extract "^2.1.2"

multicast-dns-service-types@^1.1.0:
version "1.1.0"
Expand Down Expand Up @@ -20091,6 +20133,11 @@ node-addon-api@^3.2.0:
resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-3.2.1.tgz#81325e0a2117789c0128dab65e7e38f07ceba161"
integrity sha512-mmcei9JghVNDYydghQmeDX8KoAm0FAiYyIcUt/N4nhyAipB17pllZQDOJD2fotxABnt4Mdz+dKTO7eftLg4d0A==

node-addon-api@^4.3.0:
version "4.3.0"
resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-4.3.0.tgz#52a1a0b475193e0928e98e0426a0d1254782b77f"
integrity sha512-73sE9+3UaLYYFmDsFZnqCInzPyh3MqIwZO9cw58yIqAZhONrrabrYyYe3TuIqtIiOuTXVhsGau8hcrhhwSsDIQ==

node-bitmap@0.0.1:
version "0.0.1"
resolved "https://registry.yarnpkg.com/node-bitmap/-/node-bitmap-0.0.1.tgz#180eac7003e0c707618ef31368f62f84b2a69091"
Expand Down Expand Up @@ -20127,6 +20174,11 @@ node-forge@^1.2.1, node-forge@^1.3.1:
resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3"
integrity sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==

node-gyp-build-optional-packages@5.0.3:
version "5.0.3"
resolved "https://registry.yarnpkg.com/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.3.tgz#92a89d400352c44ad3975010368072b41ad66c17"
integrity sha512-k75jcVzk5wnnc/FMxsf4udAoTEUv2jY3ycfdSd3yWu6Cnd1oee6/CfZJApyscA4FJOmdoixWwiwOyf16RzD5JA==

node-gyp-build@^4.2.3:
version "4.2.3"
resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.2.3.tgz#ce6277f853835f718829efb47db20f3e4d9c4739"
Expand Down Expand Up @@ -20778,10 +20830,10 @@ ora@^5.3.0:
strip-ansi "^6.0.0"
wcwidth "^1.0.1"

ordered-binary@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/ordered-binary/-/ordered-binary-1.0.0.tgz#4f7485186b12aa42b99011aeb7aa272991d6a487"
integrity sha512-0RMlzqix3YAOZKMoXv97OIvHlqJxnmIzihjShVkYNV3JuzHbqeBOOP7wpz6yo4af1ZFnOHGsh8RK77ZmaBY3Lg==
ordered-binary@^1.4.0:
version "1.4.0"
resolved "https://registry.yarnpkg.com/ordered-binary/-/ordered-binary-1.4.0.tgz#6bb53d44925f3b8afc33d1eed0fa15693b211389"
integrity sha512-EHQ/jk4/a9hLupIKxTfUsQRej1Yd/0QLQs3vGvIqg5ZtCYSzNhkzHoZc7Zf4e4kUlDaC3Uw8Q/1opOLNN2OKRQ==

ordered-read-streams@^1.0.0:
version "1.0.1"
Expand Down Expand Up @@ -28782,10 +28834,10 @@ wcwidth@^1.0.1:
dependencies:
defaults "^1.0.3"

weak-lru-cache@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/weak-lru-cache/-/weak-lru-cache-1.0.0.tgz#f1394721169883488c554703704fbd91cda05ddf"
integrity sha512-135bPugHHIJLNx20guHgk4etZAbd7nou34NQfdKkJPgMuC3Oqn4cT6f7ORVvnud9oEyXJVJXPcTFsUvttGm5xg==
weak-lru-cache@^1.2.2:
version "1.2.2"
resolved "https://registry.yarnpkg.com/weak-lru-cache/-/weak-lru-cache-1.2.2.tgz#fdbb6741f36bae9540d12f480ce8254060dccd19"
integrity sha512-DEAoo25RfSYMuTGc9vPJzZcZullwIqRDSI9LOy+fkCJPi6hykCnfKaXTuPBDuXAUcqHXyOgFtHNp/kB2FjYHbw==

web-namespaces@^1.0.0:
version "1.1.4"
Expand Down

0 comments on commit e7de50f

Please sign in to comment.