diff --git a/package.json b/package.json index addbf8714e8725..15d686871ef1cd 100644 --- a/package.json +++ b/package.json @@ -757,7 +757,7 @@ "jsondiffpatch": "0.4.1", "license-checker": "^16.0.0", "listr": "^0.14.1", - "lmdb-store": "^1.6.11", + "lmdb": "^2.6.9", "marge": "^1.0.1", "micromatch": "3.1.10", "minimist": "^1.2.6", diff --git a/packages/kbn-optimizer/BUILD.bazel b/packages/kbn-optimizer/BUILD.bazel index 485e5f1044aa32..5c798ea57a03cf 100644 --- a/packages/kbn-optimizer/BUILD.bazel +++ b/packages/kbn-optimizer/BUILD.bazel @@ -46,7 +46,7 @@ RUNTIME_DEPS = [ "@npm//execa", "@npm//jest-diff", "@npm//json-stable-stringify", - "@npm//lmdb-store", + "@npm//lmdb", "@npm//loader-utils", "@npm//node-sass", "@npm//normalize-path", @@ -75,7 +75,7 @@ TYPES_DEPS = [ "@npm//del", "@npm//execa", "@npm//jest-diff", - "@npm//lmdb-store", + "@npm//lmdb", "@npm//pirates", "@npm//resize-observer-polyfill", "@npm//rxjs", diff --git a/packages/kbn-optimizer/src/node/cache.ts b/packages/kbn-optimizer/src/node/cache.ts index 1320d275022a91..db7e7f3fb69a97 100644 --- a/packages/kbn-optimizer/src/node/cache.ts +++ b/packages/kbn-optimizer/src/node/cache.ts @@ -10,7 +10,7 @@ import Path from 'path'; import { Writable } from 'stream'; import chalk from 'chalk'; -import * as LmdbStore from 'lmdb-store'; +import * as LmdbStore from 'lmdb'; const GLOBAL_ATIME = `${Date.now()}`; const MINUTE = 1000 * 60; @@ -44,6 +44,10 @@ export class Cache { name: 'codes', encoding: 'string', maxReaders: 500, + overlappingSync: false, + noSync: true, + noMetaSync: true, + noMemInit: true, }); // TODO: redundant 'name' syntax is necessary because of a bug that I have yet to fix @@ -66,8 +70,8 @@ export class Cache { // keys which haven't been used in 30 days. We use `unref()` to // make sure this timer doesn't hold other processes open // unexpectedly - this.timer = setTimeout(() => { - this.pruneOldKeys(); + this.timer = setTimeout(async () => { + await this.pruneOldKeys(); }, 30 * MINUTE); // timer.unref is not defined in jest which emulates the dom by default @@ -104,12 +108,10 @@ export class Cache { async update(path: string, file: { mtime: string; code: string; map: any }) { const key = this.getKey(path); - await Promise.all([ - this.safePut(this.atimes, key, GLOBAL_ATIME), - this.safePut(this.mtimes, key, file.mtime), - this.safePut(this.codes, key, file.code), - this.safePut(this.sourceMaps, key, JSON.stringify(file.map)), - ]); + this.safePut(this.atimes, key, GLOBAL_ATIME); + this.safePut(this.mtimes, key, file.mtime); + this.safePut(this.codes, key, file.code); + this.safePut(this.sourceMaps, key, JSON.stringify(file.map)); } close() { @@ -135,9 +137,9 @@ export class Cache { } } - private async safePut(db: LmdbStore.Database, key: string, value: V) { + private safePut(db: LmdbStore.Database, key: string, value: V) { try { - await db.put(key, value); + db.putSync(key, value); this.debug('PUT', db, key); } catch (error) { this.logError('PUT', db, key, error); @@ -167,7 +169,6 @@ export class Cache { const validKeys: string[] = []; const invalidKeys: string[] = []; - // @ts-expect-error See https://github.com/DoctorEvidence/lmdb-store/pull/18 for (const { key, value } of this.atimes.getRange()) { const atime = parseInt(`${value}`, 10); if (Number.isNaN(atime) || atime < ATIME_LIMIT) { diff --git a/packages/kbn-optimizer/src/node/integration_tests/cache.test.ts b/packages/kbn-optimizer/src/node/integration_tests/cache.test.ts index 8ce58aa74214f7..c12a79be37633f 100644 --- a/packages/kbn-optimizer/src/node/integration_tests/cache.test.ts +++ b/packages/kbn-optimizer/src/node/integration_tests/cache.test.ts @@ -81,6 +81,7 @@ it('returns undefined until values are set', async () => { PUT [sourceMaps] prefix:bar.js HIT [mtimes] prefix:bar.js HIT [codes] prefix:bar.js + PUT [atimes] prefix:bar.js HIT [sourceMaps] prefix:bar.js " `); diff --git a/yarn.lock b/yarn.lock index b93b343e617479..83128a83c0f027 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2881,6 +2881,36 @@ resolved "https://registry.yarnpkg.com/@kwsites/promise-deferred/-/promise-deferred-1.1.1.tgz#8ace5259254426ccef57f3175bc64ed7095ed919" integrity sha512-GaHYm+c0O9MjZRu0ongGBRbinu8gVAMd2UZjji6jVmqKtZluZnptXGWhz1E8j8D2HJ3f/yMxKAUC0b+57wncIw== +"@lmdb/lmdb-darwin-arm64@2.6.9": + version "2.6.9" + resolved "https://registry.yarnpkg.com/@lmdb/lmdb-darwin-arm64/-/lmdb-darwin-arm64-2.6.9.tgz#4b84bb0ad71e78472332920c9cf8603ea3dad0bc" + integrity sha512-QxyheKfTP9k5ZVAiddCqGUtp2AD3/BMgYfski96iIbFH0skPFO+MYARMGZuemTgyM9uieT+5oKj4FjigWplpWg== + +"@lmdb/lmdb-darwin-x64@2.6.9": + version "2.6.9" + resolved "https://registry.yarnpkg.com/@lmdb/lmdb-darwin-x64/-/lmdb-darwin-x64-2.6.9.tgz#28b191a9f7a1f30462d8d179cd05598fa66ebbfc" + integrity sha512-zJ1oUepZMaqiujvWeWJRG5VHXBS3opJnjAzbd4vTVsQFT0t5rbPhHgAJ2ruR9rVrb2V1BINJZuwpjhIOg9fLCQ== + +"@lmdb/lmdb-linux-arm64@2.6.9": + version "2.6.9" + resolved "https://registry.yarnpkg.com/@lmdb/lmdb-linux-arm64/-/lmdb-linux-arm64-2.6.9.tgz#274dfe11209a70c059cb55c72026c24903dde3e1" + integrity sha512-KZRet8POwKowbYZqrRqdYJ+B6l+7cWG18vMCe2sgOSuE41sEMpfRQ1mKcolt3fsr0KVbuP63aPG+dwi0wGpX9w== + +"@lmdb/lmdb-linux-arm@2.6.9": + version "2.6.9" + resolved "https://registry.yarnpkg.com/@lmdb/lmdb-linux-arm/-/lmdb-linux-arm-2.6.9.tgz#7bacd104067e7dbb1bb67c907c1bc642e2d2ac96" + integrity sha512-Umw+ikxbsYZHHqr8eMycmApj6IIZCK4k1rp5/pqqx9FvAaPv4/Y63owiMLoKfipjel0YPaNyvSeXAJK3l/8Pbw== + +"@lmdb/lmdb-linux-x64@2.6.9": + version "2.6.9" + resolved "https://registry.yarnpkg.com/@lmdb/lmdb-linux-x64/-/lmdb-linux-x64-2.6.9.tgz#d37b25c9b553c5d5e66055a64d118e3fd42557d9" + integrity sha512-11xFQ4kCIPGnYULcfkW4SIMIY1sukA4DHez62DKvYn+tLr4AB1o9jm1Jk6bisKFh5Cql+JUr7klHxeIuxvGZdg== + +"@lmdb/lmdb-win32-x64@2.6.9": + version "2.6.9" + resolved "https://registry.yarnpkg.com/@lmdb/lmdb-win32-x64/-/lmdb-win32-x64-2.6.9.tgz#bf8e647dabd8b672744315f5df3e363b5987a463" + integrity sha512-qECZ+1j3PSarYeCmJlYlrxq3TB7S020ICrYmpxyQyphbRiMI9I1Bw4t+vPrMAEKsTqB8UaOzBp21YWUpsiCBfA== + "@loaders.gl/core@2.3.1", "@loaders.gl/core@^2.3.1": version "2.3.1" resolved "https://registry.yarnpkg.com/@loaders.gl/core/-/core-2.3.1.tgz#147037e17b014528dce00187aac0ec6ccb05938b" @@ -18294,17 +18324,23 @@ listr@^0.14.1: p-map "^2.0.0" rxjs "^6.3.3" -lmdb-store@^1.6.11: - version "1.6.11" - resolved "https://registry.yarnpkg.com/lmdb-store/-/lmdb-store-1.6.11.tgz#801da597af8c7a01c81f87d5cc7a7497e381236d" - integrity sha512-hIvoGmHGsFhb2VRCmfhodA/837ULtJBwRHSHKIzhMB7WtPH6BRLPsvXp1MwD3avqGzuZfMyZDUp3tccLvr721Q== - dependencies: - nan "^2.14.2" - node-gyp-build "^4.2.3" - ordered-binary "^1.0.0" - weak-lru-cache "^1.0.0" +lmdb@^2.6.9: + version "2.6.9" + resolved "https://registry.yarnpkg.com/lmdb/-/lmdb-2.6.9.tgz#aa782ec873bcf70333b251eede9e711819ef5765" + integrity sha512-rVA3OchNoKxoD2rYhtc9nooqbJmId+vvfPzTWhanRPhdVr0hbgnF9uB9ZEHFU2lEeYVdh83Pt2H6DudeWuz+JA== + dependencies: + msgpackr "1.7.2" + node-addon-api "^4.3.0" + node-gyp-build-optional-packages "5.0.3" + ordered-binary "^1.4.0" + weak-lru-cache "^1.2.2" optionalDependencies: - msgpackr "^1.4.7" + "@lmdb/lmdb-darwin-arm64" "2.6.9" + "@lmdb/lmdb-darwin-x64" "2.6.9" + "@lmdb/lmdb-linux-arm" "2.6.9" + "@lmdb/lmdb-linux-arm64" "2.6.9" + "@lmdb/lmdb-linux-x64" "2.6.9" + "@lmdb/lmdb-win32-x64" "2.6.9" load-json-file@^1.0.0: version "1.1.0" @@ -19785,20 +19821,26 @@ ms@2.1.3, ms@^2.1.3: resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== -msgpackr-extract@^1.0.14: - version "1.0.14" - resolved "https://registry.yarnpkg.com/msgpackr-extract/-/msgpackr-extract-1.0.14.tgz#87d3fe825d226e7f3d9fe136375091137f958561" - integrity sha512-t8neMf53jNZRF+f0H9VvEUVvtjGZ21odSBRmFfjZiyxr9lKYY0mpY3kSWZAIc7YWXtCZGOvDQVx2oqcgGiRBrw== +msgpackr-extract@^2.1.2: + version "2.2.0" + resolved "https://registry.yarnpkg.com/msgpackr-extract/-/msgpackr-extract-2.2.0.tgz#4bb749b58d9764cfdc0d91c7977a007b08e8f262" + integrity sha512-0YcvWSv7ZOGl9Od6Y5iJ3XnPww8O7WLcpYMDwX+PAA/uXLDtyw94PJv9GLQV/nnp3cWlDhMoyKZIQLrx33sWog== dependencies: - nan "^2.14.2" - node-gyp-build "^4.2.3" - -msgpackr@^1.4.7: - version "1.4.7" - resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.4.7.tgz#d802ade841e7d2e873000b491cdda6574a3d5748" - integrity sha512-bhC8Ed1au3L3oHaR/fe4lk4w7PLGFcWQ5XY/Tk9N6tzDRz8YndjCG68TD8zcvYZoxNtw767eF/7VpaTpU9kf9w== + node-gyp-build-optional-packages "5.0.3" optionalDependencies: - msgpackr-extract "^1.0.14" + "@msgpackr-extract/msgpackr-extract-darwin-arm64" "2.2.0" + "@msgpackr-extract/msgpackr-extract-darwin-x64" "2.2.0" + "@msgpackr-extract/msgpackr-extract-linux-arm" "2.2.0" + "@msgpackr-extract/msgpackr-extract-linux-arm64" "2.2.0" + "@msgpackr-extract/msgpackr-extract-linux-x64" "2.2.0" + "@msgpackr-extract/msgpackr-extract-win32-x64" "2.2.0" + +msgpackr@1.7.2: + version "1.7.2" + resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.7.2.tgz#68d6debf5999d6b61abb6e7046a689991ebf7261" + integrity sha512-mWScyHTtG6TjivXX9vfIy2nBtRupaiAj0HQ2mtmpmYujAmqZmaaEVPaSZ1NKLMvicaMLFzEaMk0ManxMRg8rMQ== + optionalDependencies: + msgpackr-extract "^2.1.2" multicast-dns-service-types@^1.1.0: version "1.1.0" @@ -20091,6 +20133,11 @@ node-addon-api@^3.2.0: resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-3.2.1.tgz#81325e0a2117789c0128dab65e7e38f07ceba161" integrity sha512-mmcei9JghVNDYydghQmeDX8KoAm0FAiYyIcUt/N4nhyAipB17pllZQDOJD2fotxABnt4Mdz+dKTO7eftLg4d0A== +node-addon-api@^4.3.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-4.3.0.tgz#52a1a0b475193e0928e98e0426a0d1254782b77f" + integrity sha512-73sE9+3UaLYYFmDsFZnqCInzPyh3MqIwZO9cw58yIqAZhONrrabrYyYe3TuIqtIiOuTXVhsGau8hcrhhwSsDIQ== + node-bitmap@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/node-bitmap/-/node-bitmap-0.0.1.tgz#180eac7003e0c707618ef31368f62f84b2a69091" @@ -20127,6 +20174,11 @@ node-forge@^1.2.1, node-forge@^1.3.1: resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3" integrity sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA== +node-gyp-build-optional-packages@5.0.3: + version "5.0.3" + resolved "https://registry.yarnpkg.com/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.0.3.tgz#92a89d400352c44ad3975010368072b41ad66c17" + integrity sha512-k75jcVzk5wnnc/FMxsf4udAoTEUv2jY3ycfdSd3yWu6Cnd1oee6/CfZJApyscA4FJOmdoixWwiwOyf16RzD5JA== + node-gyp-build@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.2.3.tgz#ce6277f853835f718829efb47db20f3e4d9c4739" @@ -20778,10 +20830,10 @@ ora@^5.3.0: strip-ansi "^6.0.0" wcwidth "^1.0.1" -ordered-binary@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/ordered-binary/-/ordered-binary-1.0.0.tgz#4f7485186b12aa42b99011aeb7aa272991d6a487" - integrity sha512-0RMlzqix3YAOZKMoXv97OIvHlqJxnmIzihjShVkYNV3JuzHbqeBOOP7wpz6yo4af1ZFnOHGsh8RK77ZmaBY3Lg== +ordered-binary@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/ordered-binary/-/ordered-binary-1.4.0.tgz#6bb53d44925f3b8afc33d1eed0fa15693b211389" + integrity sha512-EHQ/jk4/a9hLupIKxTfUsQRej1Yd/0QLQs3vGvIqg5ZtCYSzNhkzHoZc7Zf4e4kUlDaC3Uw8Q/1opOLNN2OKRQ== ordered-read-streams@^1.0.0: version "1.0.1" @@ -28782,10 +28834,10 @@ wcwidth@^1.0.1: dependencies: defaults "^1.0.3" -weak-lru-cache@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/weak-lru-cache/-/weak-lru-cache-1.0.0.tgz#f1394721169883488c554703704fbd91cda05ddf" - integrity sha512-135bPugHHIJLNx20guHgk4etZAbd7nou34NQfdKkJPgMuC3Oqn4cT6f7ORVvnud9oEyXJVJXPcTFsUvttGm5xg== +weak-lru-cache@^1.2.2: + version "1.2.2" + resolved "https://registry.yarnpkg.com/weak-lru-cache/-/weak-lru-cache-1.2.2.tgz#fdbb6741f36bae9540d12f480ce8254060dccd19" + integrity sha512-DEAoo25RfSYMuTGc9vPJzZcZullwIqRDSI9LOy+fkCJPi6hykCnfKaXTuPBDuXAUcqHXyOgFtHNp/kB2FjYHbw== web-namespaces@^1.0.0: version "1.1.4"