diff --git a/packages/store/README.md b/packages/store/README.md index df750c50337..6135a8253ce 100644 --- a/packages/store/README.md +++ b/packages/store/README.md @@ -18,4 +18,38 @@ Store adds some additional functionality on top of Map. Map, because the Map methods are not tied to a particular Map instance. -See @agoric/weak-store for the wrapper around JavaScript's WeakMap abstraction. +See `makeWeakStore` for the wrapper around JavaScript's WeakMap abstraction. + +# External Store + +An External Store is defined by its maker function, and provides abstractions +that are compatible with large, synchronous secondary storage that can be paged +in and out of local memory. + +```js +import { makeExternalStore } from '@agoric/store'; + +// Here is us defining an instance store for 'hello' objects. +const estore = makeExternalStore((msg = 'Hello') => ({ + hello(nickname) { + return `${msg}, ${nickname}!`; + }, +})); + +const h = estore.makeInstance('Hi'); +h.hello('friend') === 'Hi, friend!'; +const wm = estore.makeWeakMap('Hello object'); +wm.init(h, 'data'); +// ... time passes and h is paged out and reloaded. +wm.get(h) === 'data'; +wm.set(h, 'new-data'); +// ... time passes and h is paged out and reloaded. +map.delete(h); +``` + +Note that when you import and use the `makeExternalStore` function, the platform +you are running on may rewrite your code to use a more scalable implementation +of that function. If it is not rewritten, then `makeExternalStore` will use +`makeMemoryExternalStore`, a full-featured, though in-memory-only +implementation. If you don't desire rewriting, then use +`makeMemoryExternalStore` directly. diff --git a/packages/store/package.json b/packages/store/package.json index 20a7e94adbb..440ed8f1fe5 100644 --- a/packages/store/package.json +++ b/packages/store/package.json @@ -2,13 +2,13 @@ "name": "@agoric/store", "version": "0.2.3-dev.2", "description": "Wrapper for JavaScript map", - "main": "src/store.js", + "main": "src/index.js", "engines": { "node": ">=11.0" }, "scripts": { "build": "exit 0", - "test": "exit 0", + "test": "ava", "lint-fix": "yarn lint --fix", "lint-check": "yarn lint", "lint": "yarn lint:types && eslint '**/*.js'", @@ -49,6 +49,9 @@ "env": { "es6": true }, + "globals": { + "harden": "readonly" + }, "rules": { "implicit-arrow-linebreak": "off", "function-paren-newline": "off", diff --git a/packages/store/src/external/default.js b/packages/store/src/external/default.js new file mode 100644 index 00000000000..34c31af024f --- /dev/null +++ b/packages/store/src/external/default.js @@ -0,0 +1,8 @@ +// Copyright (C) 2019 Agoric, under Apache license 2.0 + +// @ts-check + +import '../types'; +import { makeMemoryExternalStore } from './memory'; + +export const makeExternalStore = makeMemoryExternalStore; diff --git a/packages/store/src/external/hydrate.js b/packages/store/src/external/hydrate.js new file mode 100644 index 00000000000..96b25af3982 --- /dev/null +++ b/packages/store/src/external/hydrate.js @@ -0,0 +1,115 @@ +// @ts-check + +import '../types'; + +import { makeWeakStore } from '../weak-store'; +import { makeStore } from '../store'; + +/** + * @callback MakeBackingStore + * @param {HydrateHook} hydrateHook + * @returns {BackingStore} + */ + +/** + * This creates an external store maker for a given storage backend, supporting + * the Closure interface that the rewriter targets. + * + * @template {Array} A + * @template {Instance} T + * @param {MakeBackingStore} makeBackingStore + * @returns {MakeHydrateExternalStore} + */ +export const makeHydrateExternalStoreMaker = makeBackingStore => { + const serialize = JSON.stringify; + const unserialize = JSON.parse; + + /** @type {WeakStore} */ + const instanceToKey = makeWeakStore('instance'); + + let lastStoreKey = 0; + + // This has to be a strong store, since it is indexed by key. + const storeKeyToHydrate = makeStore('storeKey'); + + /** + * Create a data object that queues writes to the store. + * + * @param {HydrateData} data + * @param {() => void} markDirty + */ + const makeActiveData = (data, markDirty) => { + const activeData = {}; + // For every property in data... + for (const prop of Object.getOwnPropertyNames(data)) { + // Define a getter and setter on activeData. + Object.defineProperty(activeData, prop, { + get: () => data[prop], + set: value => { + data[prop] = value; + markDirty(); + }, + }); + } + return harden(activeData); + }; + + /** + * @type {BackingStore} + */ + let backing; + const hydrateHook = { + getKey(value) { + return instanceToKey.get(value); + }, + load([storeKey, instanceKey]) { + const hydrate = storeKeyToHydrate.get(storeKey); + const store = backing.findStore(storeKey); + + const data = unserialize(store.get(instanceKey)); + const markDirty = () => store.set(instanceKey, serialize(data)); + + const activeData = makeActiveData(data, markDirty); + const obj = hydrate(activeData); + instanceToKey.init(obj, [storeKey, instanceKey]); + return obj; + }, + drop(storeKey) { + storeKeyToHydrate.delete(storeKey); + }, + }; + + backing = makeBackingStore(hydrateHook); + + function makeHydrateExternalStore(instanceName, adaptArguments, makeHydrate) { + let lastInstanceKey = 0; + + lastStoreKey += 1; + const storeKey = `${lastStoreKey}`; + const store = backing.makeStore(storeKey, instanceName); + + const initHydrate = makeHydrate(true); + storeKeyToHydrate.init(storeKey, makeHydrate(undefined)); + + /** @type {ExternalStore<(...args: A) => T>} */ + const estore = { + makeInstance(...args) { + const data = adaptArguments(...args); + // Create a new object with the above guts. + lastInstanceKey += 1; + const instanceKey = `${lastInstanceKey}`; + initHydrate(data); + + // We store and reload it to sanity-check the initial state and also to + // ensure that the new object has active data. + store.init(instanceKey, serialize(data)); + return hydrateHook.load([storeKey, instanceKey]); + }, + makeWeakStore() { + return store.makeWeakStore(); + }, + }; + return estore; + } + return harden(makeHydrateExternalStore); +}; diff --git a/packages/store/src/external/memory.js b/packages/store/src/external/memory.js new file mode 100644 index 00000000000..722b876cb73 --- /dev/null +++ b/packages/store/src/external/memory.js @@ -0,0 +1,25 @@ +// Copyright (C) 2019-20 Agoric, under Apache license 2.0 + +// @ts-check +import { makeWeakStore } from '../weak-store'; +import '../types'; + +/** + * Create a completely in-memory "external" store. This store will be + * garbage-collected in the usual way, but it will not page out any objects to + * secondary storage. + * + * @template {(...args: any[]) => Instance} M + * @param {string} instanceName + * @param {M} maker + * @returns {ExternalStore} + */ +export function makeMemoryExternalStore(instanceName, maker) { + return harden({ + makeInstance: maker, + makeWeakStore() { + return makeWeakStore(instanceName); + }, + }); +} +harden(makeMemoryExternalStore); diff --git a/packages/store/src/index.js b/packages/store/src/index.js new file mode 100644 index 00000000000..e3f73af4093 --- /dev/null +++ b/packages/store/src/index.js @@ -0,0 +1,8 @@ +export { makeStore } from './store'; +export { makeWeakStore } from './weak-store'; +export { makeExternalStore } from './external/default'; +export { makeMemoryExternalStore } from './external/memory'; +export { makeHydrateExternalStoreMaker } from './external/hydrate'; + +// Backward compatibility. +export { makeStore as default } from './store'; diff --git a/packages/store/src/store.js b/packages/store/src/store.js index d5d01446401..c8d09b450bf 100644 --- a/packages/store/src/store.js +++ b/packages/store/src/store.js @@ -1,24 +1,9 @@ // Copyright (C) 2019 Agoric, under Apache license 2.0 -/* global harden */ // @ts-check import { assert, details, q } from '@agoric/assert'; -/** - * @template K,V - * @typedef {Object} Store - A safety wrapper around a Map - * @property {(key: K) => boolean} has - Check if a key exists - * @property {(key: K, value: V) => void} init - Initialize the key only if it doesn't already exist - * @property {(key: K) => V} get - Return a value for the key. Throws - * if not found. - * @property {(key: K, value: V) => void} set - Set the key. Throws if not found. - * @property {(key: K) => void} delete - Remove the key. Throws if not found. - * @property {() => K[]} keys - Return an array of keys - * @property {() => V[]} values - Return an array of values - * @property {() => [K, V][]} entries - Return an array of entries - */ - /** * Distinguishes between adding a new key (init) and updating or * referencing a key (get, set, delete). @@ -29,7 +14,7 @@ import { assert, details, q } from '@agoric/assert'; * @param {string} [keyName='key'] - the column name for the key * @returns {Store} */ -function makeStore(keyName = 'key') { +export function makeStore(keyName = 'key') { const store = new Map(); const assertKeyDoesNotExist = key => assert(!store.has(key), details`${q(keyName)} already registered: ${key}`); @@ -59,4 +44,3 @@ function makeStore(keyName = 'key') { }); } harden(makeStore); -export default makeStore; diff --git a/packages/store/src/types.js b/packages/store/src/types.js new file mode 100644 index 00000000000..9070b7d7230 --- /dev/null +++ b/packages/store/src/types.js @@ -0,0 +1,83 @@ +/** + * @typedef {Record} Instance + */ + +/** + * @template K,V + * @typedef {Object} Store - A safety wrapper around a Map + * @property {(key: K) => boolean} has - Check if a key exists + * @property {(key: K, value: V) => void} init - Initialize the key only if it doesn't already exist + * @property {(key: K) => V} get - Return a value for the key. Throws + * if not found. + * @property {(key: K, value: V) => void} set - Set the key. Throws if not found. + * @property {(key: K) => void} delete - Remove the key. Throws if not found. + * @property {() => K[]} keys - Return an array of keys + * @property {() => V[]} values - Return an array of values + * @property {() => [K, V][]} entries - Return an array of entries + */ + +/** + * @template K,V + * @typedef {Object} WeakStore - A safety wrapper around a WeakMap + * @property {(key: any) => boolean} has - Check if a key exists + * @property {(key: K, value: V) => void} init - Initialize the key only if it doesn't already exist + * @property {(key: any) => V} get - Return a value for the key. Throws + * if not found. + * @property {(key: K, value: V) => void} set - Set the key. Throws if not found. + * @property {(key: K) => void} delete - Remove the key. Throws if not found. + */ + +/** + * Distinguishes between adding a new key (init) and updating or + * referencing a key (get, set, delete). + * + * `init` is only allowed if the key does not already exist. `Get`, + * `set` and `delete` are only allowed if the key does already exist. + * @template K,V + * @callback MakeWeakStore + * @param {string} [keyName='key'] - the column name for the key + * @returns {WeakStore} + */ + +/** + * An external store for a given constructor. + * + * @template {(...args: Array) => Instance} C + * @typedef {Object} ExternalStore + * @property {C} makeInstance + * @property {MakeWeakStore, any>} makeWeakStore + */ + +/** + * @typedef {Record} HydrateData + */ + +/** + * @typedef {Object} HydrateHook + * @property {(value: any) => [string, string]} getKey + * @property {(key: [string, string]) => any} load + * @property {(storeKey: string) => void} drop + */ + +/** + * An external store that decouples the closure data from the returned + * "representative" instance. + * + * @template {Array} A + * @template {Instance} T + * @callback MakeHydrateExternalStore + * @param {string} instanceKind + * @param {(...args: A) => HydrateData} adaptArguments + * @param {(init: boolean | undefined) => (data: HydrateData) => T} makeHydrate + * @returns {ExternalStore<(...args: A) => T>} + */ + +/** + * @typedef {Store & { makeWeakStore: () => WeakStore }}} InstanceStore + */ + +/** + * @typedef {Object} BackingStore + * @property {(storeId: string, instanceKind: string) => InstanceStore} makeStore + * @property {(storeId: string) => InstanceStore} findStore + */ diff --git a/packages/store/src/weak-store.js b/packages/store/src/weak-store.js new file mode 100644 index 00000000000..5217c0fab79 --- /dev/null +++ b/packages/store/src/weak-store.js @@ -0,0 +1,40 @@ +// Copyright (C) 2019 Agoric, under Apache license 2.0 + +// @ts-check + +import { assert, details, q } from '@agoric/assert'; +import './types'; + +/** + * @template {Record} K + * @template {any} V + * @param {string} [keyName='key'] + * @returns {WeakStore} + */ +export function makeWeakStore(keyName = 'key') { + const wm = new WeakMap(); + const assertKeyDoesNotExist = key => + assert(!wm.has(key), details`${q(keyName)} already registered: ${key}`); + const assertKeyExists = key => + assert(wm.has(key), details`${q(keyName)} not found: ${key}`); + return harden({ + has: key => wm.has(key), + init: (key, value) => { + assertKeyDoesNotExist(key); + wm.set(key, value); + }, + get: key => { + assertKeyExists(key); + return wm.get(key); + }, + set: (key, value) => { + assertKeyExists(key); + wm.set(key, value); + }, + delete: key => { + assertKeyExists(key); + wm.delete(key); + }, + }); +} +harden(makeWeakStore); diff --git a/packages/store/test/test-external-store.js b/packages/store/test/test-external-store.js new file mode 100644 index 00000000000..e09effe4152 --- /dev/null +++ b/packages/store/test/test-external-store.js @@ -0,0 +1,131 @@ +// @ts-check +/* eslint-disable no-use-before-define */ +import '@agoric/install-ses'; +import test from 'ava'; +import { + makeStore, + makeExternalStore, + makeHydrateExternalStoreMaker, + makeWeakStore, +} from '../src/index'; + +import '../src/types'; + +const moduleLevel = 'module-level'; + +const runTests = (t, mf) => { + const h = mf('Hello'); + t.deepEqual(h.getCount(), { invocationCount: 26, moduleLevel }); + t.is(h.hello('World'), `Hello, World!`); + t.deepEqual(h.getCount(), { invocationCount: 27, moduleLevel }); + t.is(h.hello('second'), `Hello, second!`); + t.deepEqual(h.getCount(), { invocationCount: 28, moduleLevel }); + return h; +}; + +test('original sources', t => { + // This is the original source code. + const { makeInstance: makeFoo } = makeExternalStore( + 'foo instance', + (msg = 'Hello') => { + let startCount = 24; + startCount += 1; + let invocationCount = startCount; + const obj = { + hello(nick) { + invocationCount += 1; + return `${msg}, ${nick}!`; + }, + getCount() { + return { moduleLevel, invocationCount }; + }, + }; + obj.hello('init'); + return obj; + }, + ); + + runTests(t, makeFoo); +}); + +test('rewritten code', t => { + /** @type {HydrateHook} */ + let swingSetHydrateHook; + const makeSwingSetCollection = makeHydrateExternalStoreMaker(hydrateHook => { + swingSetHydrateHook = hydrateHook; + const idToStore = makeStore('storeId'); + return { + findStore(storeId) { + return idToStore.get(storeId); + }, + makeStore(storeId, instanceKind) { + const store = makeStore(`${instanceKind} ids`); + idToStore.init(storeId, store); + return { + ...store, + makeWeakStore() { + return makeWeakStore(instanceKind); + }, + }; + }, + }; + }); + + /** + * This is the rewritten source code, line numbers can be preserved. + * + * The return value drives the analysis of which expressions need to be + * evaluated to resurrect the object. Those expressions determine which + * variables need to be captured by $hdata to write to and read from the store. + * + * Side-effecting expressions are rewritten to be conditional on whether the + * initialisation step is taking place. This is done on a per-expression + * basis, since it is known any variables that are changed by the side-effect + * are either already captured by $hdata, or aren't needed by the return + * value. + * + * Declarations are not considered side-effects. + */ + const store = makeSwingSetCollection( + 'Hello instance', + (msg = 'Hello') => ({ msg }), + $hinit => $hdata => { + let startCount = $hinit && 24; + $hinit && (startCount += 1); + $hinit && ($hdata.invocationCount = startCount); + const obj = { + hello(nick) { + $hdata.invocationCount += 1; + return `${$hdata.msg}, ${nick}!`; + }, + getCount() { + return { moduleLevel, invocationCount: $hdata.invocationCount }; + }, + }; + $hinit && obj.hello('init'); + return obj; + }, + ); + + const h = runTests(t, store.makeInstance); + const key = swingSetHydrateHook.getKey(h); + t.deepEqual(key, ['1', '1']); + const h2 = swingSetHydrateHook.load(key); + + // We get a different representative, which shares the key. + t.not(h2, h); + t.deepEqual(swingSetHydrateHook.getKey(h2), ['1', '1']); + + // The methods are there now, too. + const last = h.getCount(); + t.deepEqual(h2.getCount(), last); + h2.hello('restored'); + + // Note that the explicitly-loaded object state evolves independently. + const next = h2.getCount(); + t.deepEqual(next, { + ...last, + invocationCount: last.invocationCount + 1, + }); + t.deepEqual(h.getCount(), last); +});