Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

chore(NA): updates from lmdb-store to lmdb #145891

Merged
merged 17 commits into from
Nov 28, 2022
Merged

chore(NA): updates from lmdb-store to lmdb #145891

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
4ae455e
chore(NA): upgrade lmdb-store into lmdb@2.x
mistic Nov 16, 2022
ab25cb0
chore(NA): removes lmdb-store pkg
mistic Nov 16, 2022
b8cd5dd
chore(NA): merge and solve conflicts with main
mistic Nov 21, 2022
1b25970
Merge branch 'main' into lmdb-upgrade
mistic Nov 21, 2022
e688608
Merge branch 'main' into lmdb-upgrade
kibanamachine Nov 21, 2022
6013359
Merge remote-tracking branch 'upstream/main' into lmdb-upgrade
mistic Nov 22, 2022
923e1f1
chore(NA): introduce new cache implementation for new lmbdb version
mistic Nov 22, 2022
5e79a07
[CI] Auto-commit changed files from 'node scripts/precommit_hook.js -…
kibanamachine Nov 22, 2022
8a38a1b
test(NA): fix optimizer integration tests
mistic Nov 22, 2022
ee16b9b
chore(NA): merge and solve conflicts with main
mistic Nov 22, 2022
591a915
chore(NA): set new cache version
mistic Nov 22, 2022
11e8b11
fix(NA): calling safe put for maps if available
mistic Nov 22, 2022
a27eb92
Merge branch 'main' into lmdb-upgrade
kibanamachine Nov 22, 2022
9be2d29
Merge branch 'main' into lmdb-upgrade
kibanamachine Nov 22, 2022
e785691
Merge branch 'main' into lmdb-upgrade
mistic Nov 23, 2022
3e68532
Merge branch 'main' into lmdb-upgrade
mistic Nov 25, 2022
1a2d07d
Merge branch 'main' into lmdb-upgrade
mistic Nov 28, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1069,7 +1069,7 @@
"jsondiffpatch": "0.4.1",
"license-checker": "^25.0.1",
"listr": "^0.14.1",
"lmdb-store": "^1",
"lmdb": "^2.6.9",
"loader-utils": "^2.0.4",
"marge": "^1.0.1",
"micromatch": "^4.0.5",
Expand Down
4 changes: 2 additions & 2 deletions packages/kbn-optimizer/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ RUNTIME_DEPS = [
"@npm//execa",
"@npm//json-stable-stringify",
"@npm//js-yaml",
"@npm//lmdb-store",
"@npm//lmdb",
"@npm//loader-utils",
"@npm//node-sass",
"@npm//normalize-path",
Expand Down Expand Up @@ -88,7 +88,7 @@ TYPES_DEPS = [
"@npm//cpy",
"@npm//del",
"@npm//execa",
"@npm//lmdb-store",
"@npm//lmdb",
"@npm//pirates",
"@npm//rxjs",
"@npm//@types/babel__core",
Expand Down
29 changes: 17 additions & 12 deletions packages/kbn-optimizer/src/node/cache.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import Path from 'path';
import { Writable } from 'stream';

import chalk from 'chalk';
import * as LmdbStore from 'lmdb-store';
import * as LmdbStore from 'lmdb';

const GLOBAL_ATIME = `${Date.now()}`;
const MINUTE = 1000 * 60;
Expand Down Expand Up @@ -44,6 +44,10 @@ export class Cache {
name: 'codes',
encoding: 'string',
maxReaders: 500,
overlappingSync: false,
noSync: true,
noMetaSync: true,
noMemInit: true,
});

// TODO: redundant 'name' syntax is necessary because of a bug that I have yet to fix
Expand All @@ -66,8 +70,8 @@ export class Cache {
// keys which haven't been used in 30 days. We use `unref()` to
// make sure this timer doesn't hold other processes open
// unexpectedly
this.timer = setTimeout(() => {
this.pruneOldKeys();
this.timer = setTimeout(async () => {
await this.pruneOldKeys();
}, 30 * MINUTE);

// timer.unref is not defined in jest which emulates the dom by default
Expand Down Expand Up @@ -107,12 +111,14 @@ export class Cache {

async update(path: string, file: { mtime: string; code: string; map?: any }) {
const key = this.getKey(path);
await Promise.all([
this.safePut(this.atimes, key, GLOBAL_ATIME),
this.safePut(this.mtimes, key, file.mtime),
this.safePut(this.codes, key, file.code),
file.map != null ? this.safePut(this.sourceMaps, key, JSON.stringify(file.map)) : null,
]);

this.safePut(this.atimes, key, GLOBAL_ATIME);
this.safePut(this.mtimes, key, file.mtime);
this.safePut(this.codes, key, file.code);

if (file.map != null) {
this.safePut(this.sourceMaps, key, JSON.stringify(file.map));
}
}

private getKey(path: string) {
Expand All @@ -134,9 +140,9 @@ export class Cache {
}
}

private async safePut<V>(db: LmdbStore.Database<V, string>, key: string, value: V) {
private safePut<V>(db: LmdbStore.Database<V, string>, key: string, value: V) {
try {
await db.put(key, value);
db.putSync(key, value);
this.debug('PUT', db, key);
} catch (error) {
this.logError('PUT', db, key, error);
Expand Down Expand Up @@ -166,7 +172,6 @@ export class Cache {
const validKeys: string[] = [];
const invalidKeys: string[] = [];

// @ts-expect-error See https://github.com/DoctorEvidence/lmdb-store/pull/18
for (const { key, value } of this.atimes.getRange()) {
const atime = parseInt(`${value}`, 10);
if (Number.isNaN(atime) || atime < ATIME_LIMIT) {
Expand Down
1 change: 1 addition & 0 deletions packages/kbn-optimizer/src/node/integration_tests/cache.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ it('returns undefined until values are set', async () => {
PUT [sourceMaps] prefix:bar.js
HIT [mtimes] prefix:bar.js
HIT [codes] prefix:bar.js
PUT [atimes] prefix:bar.js
HIT [sourceMaps] prefix:bar.js
"
`);
Expand Down
2 changes: 1 addition & 1 deletion packages/kbn-optimizer/src/node/node_auto_tranpilation.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ export function registerNodeAutoTranspilation() {

const cacheDir = Path.resolve(
REPO_ROOT,
'data/node_auto_transpilation_cache_v5',
'data/node_auto_transpilation_cache_v6',
UPSTREAM_BRANCH
);

Expand Down
79 changes: 60 additions & 19 deletions yarn.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4162,6 +4162,36 @@
resolved "https://registry.yarnpkg.com/@leichtgewicht/ip-codec/-/ip-codec-2.0.4.tgz#b2ac626d6cb9c8718ab459166d4bb405b8ffa78b"
integrity sha512-Hcv+nVC0kZnQ3tD9GVu5xSMR4VVYOteQIr/hwFPVEvPdlXqgGEuRjiheChHgdM+JyqdgNcmzZOX/tnl0JOiI7A==

"@lmdb/lmdb-darwin-arm64@2.6.9":
version "2.6.9"
resolved "https://registry.yarnpkg.com/@lmdb/lmdb-darwin-arm64/-/lmdb-darwin-arm64-2.6.9.tgz#4b84bb0ad71e78472332920c9cf8603ea3dad0bc"
integrity sha512-QxyheKfTP9k5ZVAiddCqGUtp2AD3/BMgYfski96iIbFH0skPFO+MYARMGZuemTgyM9uieT+5oKj4FjigWplpWg==

"@lmdb/lmdb-darwin-x64@2.6.9":
version "2.6.9"
resolved "https://registry.yarnpkg.com/@lmdb/lmdb-darwin-x64/-/lmdb-darwin-x64-2.6.9.tgz#28b191a9f7a1f30462d8d179cd05598fa66ebbfc"
integrity sha512-zJ1oUepZMaqiujvWeWJRG5VHXBS3opJnjAzbd4vTVsQFT0t5rbPhHgAJ2ruR9rVrb2V1BINJZuwpjhIOg9fLCQ==

"@lmdb/lmdb-linux-arm64@2.6.9":
version "2.6.9"
resolved "https://registry.yarnpkg.com/@lmdb/lmdb-linux-arm64/-/lmdb-linux-arm64-2.6.9.tgz#274dfe11209a70c059cb55c72026c24903dde3e1"
integrity sha512-KZRet8POwKowbYZqrRqdYJ+B6l+7cWG18vMCe2sgOSuE41sEMpfRQ1mKcolt3fsr0KVbuP63aPG+dwi0wGpX9w==

"@lmdb/lmdb-linux-arm@2.6.9":
version "2.6.9"
resolved "https://registry.yarnpkg.com/@lmdb/lmdb-linux-arm/-/lmdb-linux-arm-2.6.9.tgz#7bacd104067e7dbb1bb67c907c1bc642e2d2ac96"
integrity sha512-Umw+ikxbsYZHHqr8eMycmApj6IIZCK4k1rp5/pqqx9FvAaPv4/Y63owiMLoKfipjel0YPaNyvSeXAJK3l/8Pbw==

"@lmdb/lmdb-linux-x64@2.6.9":
version "2.6.9"
resolved "https://registry.yarnpkg.com/@lmdb/lmdb-linux-x64/-/lmdb-linux-x64-2.6.9.tgz#d37b25c9b553c5d5e66055a64d118e3fd42557d9"
integrity sha512-11xFQ4kCIPGnYULcfkW4SIMIY1sukA4DHez62DKvYn+tLr4AB1o9jm1Jk6bisKFh5Cql+JUr7klHxeIuxvGZdg==

"@lmdb/lmdb-win32-x64@2.6.9":
version "2.6.9"
resolved "https://registry.yarnpkg.com/@lmdb/lmdb-win32-x64/-/lmdb-win32-x64-2.6.9.tgz#bf8e647dabd8b672744315f5df3e363b5987a463"
integrity sha512-qECZ+1j3PSarYeCmJlYlrxq3TB7S020ICrYmpxyQyphbRiMI9I1Bw4t+vPrMAEKsTqB8UaOzBp21YWUpsiCBfA==

"@loaders.gl/core@2.3.1":
version "2.3.1"
resolved "https://registry.yarnpkg.com/@loaders.gl/core/-/core-2.3.1.tgz#147037e17b014528dce00187aac0ec6ccb05938b"
Expand Down Expand Up @@ -18540,17 +18570,23 @@ listr@^0.14.1:
p-map "^2.0.0"
rxjs "^6.3.3"

lmdb-store@^1:
version "1.6.11"
resolved "https://registry.yarnpkg.com/lmdb-store/-/lmdb-store-1.6.11.tgz#801da597af8c7a01c81f87d5cc7a7497e381236d"
integrity sha512-hIvoGmHGsFhb2VRCmfhodA/837ULtJBwRHSHKIzhMB7WtPH6BRLPsvXp1MwD3avqGzuZfMyZDUp3tccLvr721Q==
lmdb@^2.6.9:
version "2.6.9"
resolved "https://registry.yarnpkg.com/lmdb/-/lmdb-2.6.9.tgz#aa782ec873bcf70333b251eede9e711819ef5765"
integrity sha512-rVA3OchNoKxoD2rYhtc9nooqbJmId+vvfPzTWhanRPhdVr0hbgnF9uB9ZEHFU2lEeYVdh83Pt2H6DudeWuz+JA==
dependencies:
nan "^2.14.2"
node-gyp-build "^4.2.3"
ordered-binary "^1.0.0"
weak-lru-cache "^1.0.0"
msgpackr "1.7.2"
node-addon-api "^4.3.0"
node-gyp-build-optional-packages "5.0.3"
ordered-binary "^1.4.0"
weak-lru-cache "^1.2.2"
optionalDependencies:
msgpackr "^1.4.7"
"@lmdb/lmdb-darwin-arm64" "2.6.9"
"@lmdb/lmdb-darwin-x64" "2.6.9"
"@lmdb/lmdb-linux-arm" "2.6.9"
"@lmdb/lmdb-linux-arm64" "2.6.9"
"@lmdb/lmdb-linux-x64" "2.6.9"
"@lmdb/lmdb-win32-x64" "2.6.9"

load-json-file@^1.0.0:
version "1.1.0"
Expand Down Expand Up @@ -19896,7 +19932,7 @@ ms@2.1.3, ms@^2.0.0, ms@^2.1.1, ms@^2.1.3:
resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2"
integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==

msgpackr-extract@^2.2.0:
msgpackr-extract@^2.1.2:
version "2.2.0"
resolved "https://registry.yarnpkg.com/msgpackr-extract/-/msgpackr-extract-2.2.0.tgz#4bb749b58d9764cfdc0d91c7977a007b08e8f262"
integrity sha512-0YcvWSv7ZOGl9Od6Y5iJ3XnPww8O7WLcpYMDwX+PAA/uXLDtyw94PJv9GLQV/nnp3cWlDhMoyKZIQLrx33sWog==
Expand All @@ -19910,12 +19946,12 @@ msgpackr-extract@^2.2.0:
"@msgpackr-extract/msgpackr-extract-linux-x64" "2.2.0"
"@msgpackr-extract/msgpackr-extract-win32-x64" "2.2.0"

msgpackr@^1.4.7:
version "1.8.0"
resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.8.0.tgz#6cf213e88f04c5a358c61085a42a4dbe5542de44"
integrity sha512-1Cos3r86XACdjLVY4CN8r72Cgs5lUzxSON6yb81sNZP9vC9nnBrEbu1/ldBhuR9BKejtoYV5C9UhmYUvZFJSNQ==
msgpackr@1.7.2:
version "1.7.2"
resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.7.2.tgz#68d6debf5999d6b61abb6e7046a689991ebf7261"
integrity sha512-mWScyHTtG6TjivXX9vfIy2nBtRupaiAj0HQ2mtmpmYujAmqZmaaEVPaSZ1NKLMvicaMLFzEaMk0ManxMRg8rMQ==
optionalDependencies:
msgpackr-extract "^2.2.0"
msgpackr-extract "^2.1.2"

multicast-dns@^7.2.5:
version "7.2.5"
Expand Down Expand Up @@ -19953,7 +19989,7 @@ mute-stream@0.0.8:
resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-0.0.8.tgz#1630c42b2251ff81e2a283de96a5497ea92e5e0d"
integrity sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==

nan@^2.13.2, nan@^2.14.2, nan@^2.15.0:
nan@^2.13.2, nan@^2.15.0:
version "2.15.0"
resolved "https://registry.yarnpkg.com/nan/-/nan-2.15.0.tgz#3f34a473ff18e15c1b5626b62903b5ad6e665fee"
integrity sha512-8ZtvEnA2c5aYCZYd1cvgdnU6cqwixRoYg70xPLWUws5ORTa/lnw+u4amixRS/Ac5U5mQVgp9pnlSUnbNWFaWZQ==
Expand Down Expand Up @@ -20123,6 +20159,11 @@ node-addon-api@^3.0.0:
resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-3.2.1.tgz#81325e0a2117789c0128dab65e7e38f07ceba161"
integrity sha512-mmcei9JghVNDYydghQmeDX8KoAm0FAiYyIcUt/N4nhyAipB17pllZQDOJD2fotxABnt4Mdz+dKTO7eftLg4d0A==

node-addon-api@^4.3.0:
version "4.3.0"
resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-4.3.0.tgz#52a1a0b475193e0928e98e0426a0d1254782b77f"
integrity sha512-73sE9+3UaLYYFmDsFZnqCInzPyh3MqIwZO9cw58yIqAZhONrrabrYyYe3TuIqtIiOuTXVhsGau8hcrhhwSsDIQ==

node-addon-api@^5.0.0:
version "5.0.0"
resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-5.0.0.tgz#7d7e6f9ef89043befdb20c1989c905ebde18c501"
Expand Down Expand Up @@ -20171,7 +20212,7 @@ node-gyp-build-optional-packages@5.0.3:
resolved "https://registry.yarnpkg.com/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.3.tgz#92a89d400352c44ad3975010368072b41ad66c17"
integrity sha512-k75jcVzk5wnnc/FMxsf4udAoTEUv2jY3ycfdSd3yWu6Cnd1oee6/CfZJApyscA4FJOmdoixWwiwOyf16RzD5JA==

node-gyp-build@^4.2.2, node-gyp-build@^4.2.3:
node-gyp-build@^4.2.2:
version "4.5.0"
resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.5.0.tgz#7a64eefa0b21112f89f58379da128ac177f20e40"
integrity sha512-2iGbaQBV+ITgCz76ZEjmhUKAKVf7xfY1sRl4UiKQspfZMH2h06SyhNsnSVy50cwkFQDGLyif6m/6uFXHkOZ6rg==
Expand Down Expand Up @@ -20790,7 +20831,7 @@ ora@^5.4.1:
strip-ansi "^6.0.0"
wcwidth "^1.0.1"

ordered-binary@^1.0.0:
ordered-binary@^1.4.0:
version "1.4.0"
resolved "https://registry.yarnpkg.com/ordered-binary/-/ordered-binary-1.4.0.tgz#6bb53d44925f3b8afc33d1eed0fa15693b211389"
integrity sha512-EHQ/jk4/a9hLupIKxTfUsQRej1Yd/0QLQs3vGvIqg5ZtCYSzNhkzHoZc7Zf4e4kUlDaC3Uw8Q/1opOLNN2OKRQ==
Expand Down Expand Up @@ -27538,7 +27579,7 @@ wcwidth@^1.0.1:
dependencies:
defaults "^1.0.3"

weak-lru-cache@^1.0.0:
weak-lru-cache@^1.2.2:
version "1.2.2"
resolved "https://registry.yarnpkg.com/weak-lru-cache/-/weak-lru-cache-1.2.2.tgz#fdbb6741f36bae9540d12f480ce8254060dccd19"
integrity sha512-DEAoo25RfSYMuTGc9vPJzZcZullwIqRDSI9LOy+fkCJPi6hykCnfKaXTuPBDuXAUcqHXyOgFtHNp/kB2FjYHbw==
Expand Down