Skip to content

Commit

Permalink
chore(NA): updates from lmdb-store to lmdb (#145891)
Browse files Browse the repository at this point in the history 
This PR upgrades from `lbmd-store` into `lmdb` which is the new package
and fully compatible with node `v18`.
So far my tests shows the new implementation is compatible with our
usages and I'm actually seeing a great performance boost when comparing
with main specially on subsequent calls of the same command.

This can be tested by running the following 2 times on main vs this
branch `time node scripts/kibana --config
src/cli/serve/integration_tests/__fixtures__/invalid_config.yml
--migrations.skip=true --verbose`

I verify the following on my machine

**main:**
_1st run:_ 36s
_2nd run:_ 34s

**this branch:**
_1st run:_ 35s
_2nd run:_ 6s

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
  • Loading branch information
@mistic @kibanamachine
mistic and kibanamachine authored Nov 28, 2022
1 parent 13c1b0b commit e65c652
Show file tree
Hide file tree
Showing 6 changed files with 82 additions and 35 deletions.
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1069,7 +1069,7 @@
"jsondiffpatch": "0.4.1",
"license-checker": "^25.0.1",
"listr": "^0.14.1",
"lmdb-store": "^1",
"lmdb": "^2.6.9",
"loader-utils": "^2.0.4",
"marge": "^1.0.1",
"micromatch": "^4.0.5",
Expand Down
4 changes: 2 additions & 2 deletions packages/kbn-optimizer/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ RUNTIME_DEPS = [
"@npm//execa",
"@npm//json-stable-stringify",
"@npm//js-yaml",
"@npm//lmdb-store",
"@npm//lmdb",
"@npm//loader-utils",
"@npm//node-sass",
"@npm//normalize-path",
Expand Down Expand Up @@ -88,7 +88,7 @@ TYPES_DEPS = [
"@npm//cpy",
"@npm//del",
"@npm//execa",
"@npm//lmdb-store",
"@npm//lmdb",
"@npm//pirates",
"@npm//rxjs",
"@npm//@types/babel__core",
Expand Down
29 changes: 17 additions & 12 deletions packages/kbn-optimizer/src/node/cache.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import Path from 'path';
import { Writable } from 'stream';

import chalk from 'chalk';
import * as LmdbStore from 'lmdb-store';
import * as LmdbStore from 'lmdb';

const GLOBAL_ATIME = `${Date.now()}`;
const MINUTE = 1000 * 60;
Expand Down Expand Up @@ -44,6 +44,10 @@ export class Cache {
name: 'codes',
encoding: 'string',
maxReaders: 500,
overlappingSync: false,
noSync: true,
noMetaSync: true,
noMemInit: true,
});

// TODO: redundant 'name' syntax is necessary because of a bug that I have yet to fix
Expand All @@ -66,8 +70,8 @@ export class Cache {
// keys which haven't been used in 30 days. We use `unref()` to
// make sure this timer doesn't hold other processes open
// unexpectedly
this.timer = setTimeout(() => {
this.pruneOldKeys();
this.timer = setTimeout(async () => {
await this.pruneOldKeys();
}, 30 * MINUTE);

// timer.unref is not defined in jest which emulates the dom by default
Expand Down Expand Up @@ -107,12 +111,14 @@ export class Cache {

async update(path: string, file: { mtime: string; code: string; map?: any }) {
const key = this.getKey(path);
await Promise.all([
this.safePut(this.atimes, key, GLOBAL_ATIME),
this.safePut(this.mtimes, key, file.mtime),
this.safePut(this.codes, key, file.code),
file.map != null ? this.safePut(this.sourceMaps, key, JSON.stringify(file.map)) : null,
]);

this.safePut(this.atimes, key, GLOBAL_ATIME);
this.safePut(this.mtimes, key, file.mtime);
this.safePut(this.codes, key, file.code);

if (file.map != null) {
this.safePut(this.sourceMaps, key, JSON.stringify(file.map));
}
}

private getKey(path: string) {
Expand All @@ -134,9 +140,9 @@ export class Cache {
}
}

private async safePut<V>(db: LmdbStore.Database<V, string>, key: string, value: V) {
private safePut<V>(db: LmdbStore.Database<V, string>, key: string, value: V) {
try {
await db.put(key, value);
db.putSync(key, value);
this.debug('PUT', db, key);
} catch (error) {
this.logError('PUT', db, key, error);
Expand Down Expand Up @@ -166,7 +172,6 @@ export class Cache {
const validKeys: string[] = [];
const invalidKeys: string[] = [];

// @ts-expect-error See https://github.com/DoctorEvidence/lmdb-store/pull/18
for (const { key, value } of this.atimes.getRange()) {
const atime = parseInt(`${value}`, 10);
if (Number.isNaN(atime) || atime < ATIME_LIMIT) {
Expand Down
1 change: 1 addition & 0 deletions packages/kbn-optimizer/src/node/integration_tests/cache.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ it('returns undefined until values are set', async () => {
PUT [sourceMaps] prefix:bar.js
HIT [mtimes] prefix:bar.js
HIT [codes] prefix:bar.js
PUT [atimes] prefix:bar.js
HIT [sourceMaps] prefix:bar.js
"
`);
Expand Down
2 changes: 1 addition & 1 deletion packages/kbn-optimizer/src/node/node_auto_tranpilation.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ export function registerNodeAutoTranspilation() {

const cacheDir = Path.resolve(
REPO_ROOT,
'data/node_auto_transpilation_cache_v5',
'data/node_auto_transpilation_cache_v6',
UPSTREAM_BRANCH
);

Expand Down
79 changes: 60 additions & 19 deletions yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4162,6 +4162,36 @@
resolved "https://registry.yarnpkg.com/@leichtgewicht/ip-codec/-/ip-codec-2.0.4.tgz#b2ac626d6cb9c8718ab459166d4bb405b8ffa78b"
integrity sha512-Hcv+nVC0kZnQ3tD9GVu5xSMR4VVYOteQIr/hwFPVEvPdlXqgGEuRjiheChHgdM+JyqdgNcmzZOX/tnl0JOiI7A==

"@lmdb/lmdb-darwin-arm64@2.6.9":
version "2.6.9"
resolved "https://registry.yarnpkg.com/@lmdb/lmdb-darwin-arm64/-/lmdb-darwin-arm64-2.6.9.tgz#4b84bb0ad71e78472332920c9cf8603ea3dad0bc"
integrity sha512-QxyheKfTP9k5ZVAiddCqGUtp2AD3/BMgYfski96iIbFH0skPFO+MYARMGZuemTgyM9uieT+5oKj4FjigWplpWg==

"@lmdb/lmdb-darwin-x64@2.6.9":
version "2.6.9"
resolved "https://registry.yarnpkg.com/@lmdb/lmdb-darwin-x64/-/lmdb-darwin-x64-2.6.9.tgz#28b191a9f7a1f30462d8d179cd05598fa66ebbfc"
integrity sha512-zJ1oUepZMaqiujvWeWJRG5VHXBS3opJnjAzbd4vTVsQFT0t5rbPhHgAJ2ruR9rVrb2V1BINJZuwpjhIOg9fLCQ==

"@lmdb/lmdb-linux-arm64@2.6.9":
version "2.6.9"
resolved "https://registry.yarnpkg.com/@lmdb/lmdb-linux-arm64/-/lmdb-linux-arm64-2.6.9.tgz#274dfe11209a70c059cb55c72026c24903dde3e1"
integrity sha512-KZRet8POwKowbYZqrRqdYJ+B6l+7cWG18vMCe2sgOSuE41sEMpfRQ1mKcolt3fsr0KVbuP63aPG+dwi0wGpX9w==

"@lmdb/lmdb-linux-arm@2.6.9":
version "2.6.9"
resolved "https://registry.yarnpkg.com/@lmdb/lmdb-linux-arm/-/lmdb-linux-arm-2.6.9.tgz#7bacd104067e7dbb1bb67c907c1bc642e2d2ac96"
integrity sha512-Umw+ikxbsYZHHqr8eMycmApj6IIZCK4k1rp5/pqqx9FvAaPv4/Y63owiMLoKfipjel0YPaNyvSeXAJK3l/8Pbw==

"@lmdb/lmdb-linux-x64@2.6.9":
version "2.6.9"
resolved "https://registry.yarnpkg.com/@lmdb/lmdb-linux-x64/-/lmdb-linux-x64-2.6.9.tgz#d37b25c9b553c5d5e66055a64d118e3fd42557d9"
integrity sha512-11xFQ4kCIPGnYULcfkW4SIMIY1sukA4DHez62DKvYn+tLr4AB1o9jm1Jk6bisKFh5Cql+JUr7klHxeIuxvGZdg==

"@lmdb/lmdb-win32-x64@2.6.9":
version "2.6.9"
resolved "https://registry.yarnpkg.com/@lmdb/lmdb-win32-x64/-/lmdb-win32-x64-2.6.9.tgz#bf8e647dabd8b672744315f5df3e363b5987a463"
integrity sha512-qECZ+1j3PSarYeCmJlYlrxq3TB7S020ICrYmpxyQyphbRiMI9I1Bw4t+vPrMAEKsTqB8UaOzBp21YWUpsiCBfA==

"@loaders.gl/core@2.3.1":
version "2.3.1"
resolved "https://registry.yarnpkg.com/@loaders.gl/core/-/core-2.3.1.tgz#147037e17b014528dce00187aac0ec6ccb05938b"
Expand Down Expand Up @@ -18540,17 +18570,23 @@ listr@^0.14.1:
p-map "^2.0.0"
rxjs "^6.3.3"

lmdb-store@^1:
version "1.6.11"
resolved "https://registry.yarnpkg.com/lmdb-store/-/lmdb-store-1.6.11.tgz#801da597af8c7a01c81f87d5cc7a7497e381236d"
integrity sha512-hIvoGmHGsFhb2VRCmfhodA/837ULtJBwRHSHKIzhMB7WtPH6BRLPsvXp1MwD3avqGzuZfMyZDUp3tccLvr721Q==
lmdb@^2.6.9:
version "2.6.9"
resolved "https://registry.yarnpkg.com/lmdb/-/lmdb-2.6.9.tgz#aa782ec873bcf70333b251eede9e711819ef5765"
integrity sha512-rVA3OchNoKxoD2rYhtc9nooqbJmId+vvfPzTWhanRPhdVr0hbgnF9uB9ZEHFU2lEeYVdh83Pt2H6DudeWuz+JA==
dependencies:
nan "^2.14.2"
node-gyp-build "^4.2.3"
ordered-binary "^1.0.0"
weak-lru-cache "^1.0.0"
msgpackr "1.7.2"
node-addon-api "^4.3.0"
node-gyp-build-optional-packages "5.0.3"
ordered-binary "^1.4.0"
weak-lru-cache "^1.2.2"
optionalDependencies:
msgpackr "^1.4.7"
"@lmdb/lmdb-darwin-arm64" "2.6.9"
"@lmdb/lmdb-darwin-x64" "2.6.9"
"@lmdb/lmdb-linux-arm" "2.6.9"
"@lmdb/lmdb-linux-arm64" "2.6.9"
"@lmdb/lmdb-linux-x64" "2.6.9"
"@lmdb/lmdb-win32-x64" "2.6.9"

load-json-file@^1.0.0:
version "1.1.0"
Expand Down Expand Up @@ -19896,7 +19932,7 @@ ms@2.1.3, ms@^2.0.0, ms@^2.1.1, ms@^2.1.3:
resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2"
integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==

msgpackr-extract@^2.2.0:
msgpackr-extract@^2.1.2:
version "2.2.0"
resolved "https://registry.yarnpkg.com/msgpackr-extract/-/msgpackr-extract-2.2.0.tgz#4bb749b58d9764cfdc0d91c7977a007b08e8f262"
integrity sha512-0YcvWSv7ZOGl9Od6Y5iJ3XnPww8O7WLcpYMDwX+PAA/uXLDtyw94PJv9GLQV/nnp3cWlDhMoyKZIQLrx33sWog==
Expand All @@ -19910,12 +19946,12 @@ msgpackr-extract@^2.2.0:
"@msgpackr-extract/msgpackr-extract-linux-x64" "2.2.0"
"@msgpackr-extract/msgpackr-extract-win32-x64" "2.2.0"

msgpackr@^1.4.7:
version "1.8.0"
resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.8.0.tgz#6cf213e88f04c5a358c61085a42a4dbe5542de44"
integrity sha512-1Cos3r86XACdjLVY4CN8r72Cgs5lUzxSON6yb81sNZP9vC9nnBrEbu1/ldBhuR9BKejtoYV5C9UhmYUvZFJSNQ==
msgpackr@1.7.2:
version "1.7.2"
resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.7.2.tgz#68d6debf5999d6b61abb6e7046a689991ebf7261"
integrity sha512-mWScyHTtG6TjivXX9vfIy2nBtRupaiAj0HQ2mtmpmYujAmqZmaaEVPaSZ1NKLMvicaMLFzEaMk0ManxMRg8rMQ==
optionalDependencies:
msgpackr-extract "^2.2.0"
msgpackr-extract "^2.1.2"

multicast-dns@^7.2.5:
version "7.2.5"
Expand Down Expand Up @@ -19953,7 +19989,7 @@ mute-stream@0.0.8:
resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-0.0.8.tgz#1630c42b2251ff81e2a283de96a5497ea92e5e0d"
integrity sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==

nan@^2.13.2, nan@^2.14.2, nan@^2.15.0:
nan@^2.13.2, nan@^2.15.0:
version "2.15.0"
resolved "https://registry.yarnpkg.com/nan/-/nan-2.15.0.tgz#3f34a473ff18e15c1b5626b62903b5ad6e665fee"
integrity sha512-8ZtvEnA2c5aYCZYd1cvgdnU6cqwixRoYg70xPLWUws5ORTa/lnw+u4amixRS/Ac5U5mQVgp9pnlSUnbNWFaWZQ==
Expand Down Expand Up @@ -20123,6 +20159,11 @@ node-addon-api@^3.0.0:
resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-3.2.1.tgz#81325e0a2117789c0128dab65e7e38f07ceba161"
integrity sha512-mmcei9JghVNDYydghQmeDX8KoAm0FAiYyIcUt/N4nhyAipB17pllZQDOJD2fotxABnt4Mdz+dKTO7eftLg4d0A==

node-addon-api@^4.3.0:
version "4.3.0"
resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-4.3.0.tgz#52a1a0b475193e0928e98e0426a0d1254782b77f"
integrity sha512-73sE9+3UaLYYFmDsFZnqCInzPyh3MqIwZO9cw58yIqAZhONrrabrYyYe3TuIqtIiOuTXVhsGau8hcrhhwSsDIQ==

node-addon-api@^5.0.0:
version "5.0.0"
resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-5.0.0.tgz#7d7e6f9ef89043befdb20c1989c905ebde18c501"
Expand Down Expand Up @@ -20171,7 +20212,7 @@ node-gyp-build-optional-packages@5.0.3:
resolved "https://registry.yarnpkg.com/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.3.tgz#92a89d400352c44ad3975010368072b41ad66c17"
integrity sha512-k75jcVzk5wnnc/FMxsf4udAoTEUv2jY3ycfdSd3yWu6Cnd1oee6/CfZJApyscA4FJOmdoixWwiwOyf16RzD5JA==

node-gyp-build@^4.2.2, node-gyp-build@^4.2.3:
node-gyp-build@^4.2.2:
version "4.5.0"
resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.5.0.tgz#7a64eefa0b21112f89f58379da128ac177f20e40"
integrity sha512-2iGbaQBV+ITgCz76ZEjmhUKAKVf7xfY1sRl4UiKQspfZMH2h06SyhNsnSVy50cwkFQDGLyif6m/6uFXHkOZ6rg==
Expand Down Expand Up @@ -20790,7 +20831,7 @@ ora@^5.4.1:
strip-ansi "^6.0.0"
wcwidth "^1.0.1"

ordered-binary@^1.0.0:
ordered-binary@^1.4.0:
version "1.4.0"
resolved "https://registry.yarnpkg.com/ordered-binary/-/ordered-binary-1.4.0.tgz#6bb53d44925f3b8afc33d1eed0fa15693b211389"
integrity sha512-EHQ/jk4/a9hLupIKxTfUsQRej1Yd/0QLQs3vGvIqg5ZtCYSzNhkzHoZc7Zf4e4kUlDaC3Uw8Q/1opOLNN2OKRQ==
Expand Down Expand Up @@ -27538,7 +27579,7 @@ wcwidth@^1.0.1:
dependencies:
defaults "^1.0.3"

weak-lru-cache@^1.0.0:
weak-lru-cache@^1.2.2:
version "1.2.2"
resolved "https://registry.yarnpkg.com/weak-lru-cache/-/weak-lru-cache-1.2.2.tgz#fdbb6741f36bae9540d12f480ce8254060dccd19"
integrity sha512-DEAoo25RfSYMuTGc9vPJzZcZullwIqRDSI9LOy+fkCJPi6hykCnfKaXTuPBDuXAUcqHXyOgFtHNp/kB2FjYHbw==
Expand Down

0 comments on commit e65c652

Please sign in to comment.