feat(store): implement external store machinery

packages/store/README.md

@@ -18,4 +18,38 @@ Store adds some additional functionality on top of Map.
    Map, because the Map methods are not tied to a particular
    Map instance.
 
-See @agoric/weak-store for the wrapper around JavaScript's WeakMap abstraction.
+See `makeWeakStore` for the wrapper around JavaScript's WeakMap abstraction.
+
+# External Store
+
+An External Store is defined by its maker function, and provides abstractions
+that are compatible with large, synchronous secondary storage that can be paged
+in and out of local memory.
+
+```js
+import { makeExternalStore } from '@agoric/store';
+
+// Here is us defining an instance store for 'hello' objects.
+const estore = makeExternalStore((msg = 'Hello') => ({
+  hello(nickname) {
+    return `${msg}, ${nickname}!`;
+  },
+}));
+
+const h = estore.makeInstance('Hi');
+h.hello('friend') === 'Hi, friend!';
+const wm = estore.makeWeakMap('Hello object');
+wm.init(h, 'data');
+// ... time passes and h is paged out and reloaded.
+wm.get(h) === 'data';
+wm.set(h, 'new-data');
+// ... time passes and h is paged out and reloaded.
+map.delete(h);
+```
+
+Note that when you import and use the `makeExternalStore` function, the platform
+you are running on may rewrite your code to use a more scalable implementation
+of that function.  If it is not rewritten, then `makeExternalStore` will use
+`makeMemoryExternalStore`, a full-featured, though in-memory-only
+implementation.  If you don't desire rewriting, then use
+`makeMemoryExternalStore` directly.

packages/store/package.json

@@ -2,13 +2,13 @@
   "name": "@agoric/store",
   "version": "0.2.3-dev.2",
   "description": "Wrapper for JavaScript map",
-  "main": "src/store.js",
+  "main": "src/index.js",
   "engines": {
     "node": ">=11.0"
   },
   "scripts": {
     "build": "exit 0",
-    "test": "exit 0",
+    "test": "ava",
     "lint-fix": "yarn lint --fix",
     "lint-check": "yarn lint",
     "lint": "yarn lint:types && eslint '**/*.js'",
@@ -49,6 +49,9 @@
     "env": {
       "es6": true
     },
+    "globals": {
+      "harden": "readonly"
+    },
     "rules": {
       "implicit-arrow-linebreak": "off",
       "function-paren-newline": "off",

packages/store/src/external/default.js

@@ -0,0 +1,8 @@
+// Copyright (C) 2019 Agoric, under Apache license 2.0
+
+// @ts-check
+
+import '../types';
+import { makeMemoryExternalStore } from './memory';
+
+export const makeExternalStore = makeMemoryExternalStore;

packages/store/src/external/hydrate.js

@@ -0,0 +1,115 @@
+// @ts-check
+
+import '../types';
+
+import { makeWeakStore } from '../weak-store';
+import { makeStore } from '../store';
+
+/**
+ * @callback MakeBackingStore
+ * @param {HydrateHook} hydrateHook
+ * @returns {BackingStore}
+ */
+
+/**
+ * This creates an external store maker for a given storage backend, supporting
+ * the Closure interface that the rewriter targets.
+ *
+ * @template {Array<any>} A
+ * @template {Instance} T
+ * @param {MakeBackingStore} makeBackingStore
+ * @returns {MakeHydrateExternalStore<A, T>}
+ */
+export const makeHydrateExternalStoreMaker = makeBackingStore => {
+  const serialize = JSON.stringify;
+  const unserialize = JSON.parse;
+
+  /** @type {WeakStore<T, [string, string]>} */
+  const instanceToKey = makeWeakStore('instance');
+
+  let lastStoreKey = 0;
+
+  // This has to be a strong store, since it is indexed by key.
+  const storeKeyToHydrate = makeStore('storeKey');
+
+  /**
+   * Create a data object that queues writes to the store.
+   *
+   * @param {HydrateData} data
+   * @param {() => void} markDirty
+   */
+  const makeActiveData = (data, markDirty) => {
+    const activeData = {};
+    // For every property in data...
+    for (const prop of Object.getOwnPropertyNames(data)) {
+      // Define a getter and setter on activeData.
+      Object.defineProperty(activeData, prop, {
+        get: () => data[prop],
+        set: value => {
+          data[prop] = value;
+          markDirty();
+        },
+      });
+    }
+    return harden(activeData);
+  };
+
+  /**
+   * @type {BackingStore}
+   */
+  let backing;
+  const hydrateHook = {
+    getKey(value) {
+      return instanceToKey.get(value);
+    },
+    load([storeKey, instanceKey]) {
+      const hydrate = storeKeyToHydrate.get(storeKey);
+      const store = backing.findStore(storeKey);
+
+      const data = unserialize(store.get(instanceKey));
+      const markDirty = () => store.set(instanceKey, serialize(data));
+
+      const activeData = makeActiveData(data, markDirty);
+      const obj = hydrate(activeData);
+      instanceToKey.init(obj, [storeKey, instanceKey]);
+      return obj;
+    },
+    drop(storeKey) {
+      storeKeyToHydrate.delete(storeKey);
+    },
+  };
+
+  backing = makeBackingStore(hydrateHook);
+
+  function makeHydrateExternalStore(instanceName, adaptArguments, makeHydrate) {
+    let lastInstanceKey = 0;
+
+    lastStoreKey += 1;
+    const storeKey = `${lastStoreKey}`;
+    const store = backing.makeStore(storeKey, instanceName);
+
+    const initHydrate = makeHydrate(true);
+    storeKeyToHydrate.init(storeKey, makeHydrate(undefined));
+
+    /** @type {ExternalStore<(...args: A) => T>} */
+    const estore = {
+      makeInstance(...args) {
+        const data = adaptArguments(...args);
+        // Create a new object with the above guts.
+        lastInstanceKey += 1;
+        const instanceKey = `${lastInstanceKey}`;
+        initHydrate(data);
+
+        // We store and reload it to sanity-check the initial state and also to
+        // ensure that the new object has active data.
+        store.init(instanceKey, serialize(data));
+        return hydrateHook.load([storeKey, instanceKey]);
+      },
+      makeWeakStore() {
+        return store.makeWeakStore();
+      },
+    };
+    return estore;
+  }
+  return harden(makeHydrateExternalStore);
+};

packages/store/src/external/memory.js

@@ -0,0 +1,25 @@
+// Copyright (C) 2019-20 Agoric, under Apache license 2.0
+
+// @ts-check
+import { makeWeakStore } from '../weak-store';
+import '../types';
+
+/**
+ * Create a completely in-memory "external" store.  This store will be
+ * garbage-collected in the usual way, but it will not page out any objects to
+ * secondary storage.
+ *
+ * @template {(...args: any[]) => Instance} M
+ * @param {string} instanceName
+ * @param {M} maker
+ * @returns {ExternalStore<M>}
+ */
+export function makeMemoryExternalStore(instanceName, maker) {
+  return harden({
+    makeInstance: maker,
+    makeWeakStore() {
+      return makeWeakStore(instanceName);
+    },
+  });
+}
+harden(makeMemoryExternalStore);

packages/store/src/index.js

@@ -0,0 +1,8 @@
+export { makeStore } from './store';
+export { makeWeakStore } from './weak-store';
+export { makeExternalStore } from './external/default';
+export { makeMemoryExternalStore } from './external/memory';
+export { makeHydrateExternalStoreMaker } from './external/hydrate';
+
+// Backward compatibility.
+export { makeStore as default } from './store';

packages/store/src/store.js

@@ -1,24 +1,9 @@
 // Copyright (C) 2019 Agoric, under Apache license 2.0
 
-/* global harden */
 // @ts-check
 
 import { assert, details, q } from '@agoric/assert';
 
-/**
- * @template K,V
- * @typedef {Object} Store - A safety wrapper around a Map
- * @property {(key: K) => boolean} has - Check if a key exists
- * @property {(key: K, value: V) => void} init - Initialize the key only if it doesn't already exist
- * @property {(key: K) => V} get - Return a value for the key. Throws
- * if not found.
- * @property {(key: K, value: V) => void} set - Set the key. Throws if not found.
- * @property {(key: K) => void} delete - Remove the key. Throws if not found.
- * @property {() => K[]} keys - Return an array of keys
- * @property {() => V[]} values - Return an array of values
- * @property {() => [K, V][]} entries - Return an array of entries
- */
-
 /**
  * Distinguishes between adding a new key (init) and updating or
  * referencing a key (get, set, delete).
@@ -29,7 +14,7 @@ import { assert, details, q } from '@agoric/assert';
  * @param  {string} [keyName='key'] - the column name for the key
  * @returns {Store<K,V>}
  */
-function makeStore(keyName = 'key') {
+export function makeStore(keyName = 'key') {
   const store = new Map();
   const assertKeyDoesNotExist = key =>
     assert(!store.has(key), details`${q(keyName)} already registered: ${key}`);
@@ -59,4 +44,3 @@ function makeStore(keyName = 'key') {
   });
 }
 harden(makeStore);
-export default makeStore;

packages/store/src/types.js

@@ -0,0 +1,83 @@
+/**
+ * @typedef {Record<string, Function>} Instance
+ */
+
+/**
+ * @template K,V
+ * @typedef {Object} Store - A safety wrapper around a Map
+ * @property {(key: K) => boolean} has - Check if a key exists
+ * @property {(key: K, value: V) => void} init - Initialize the key only if it doesn't already exist
+ * @property {(key: K) => V} get - Return a value for the key. Throws
+ * if not found.
+ * @property {(key: K, value: V) => void} set - Set the key. Throws if not found.
+ * @property {(key: K) => void} delete - Remove the key. Throws if not found.
+ * @property {() => K[]} keys - Return an array of keys
+ * @property {() => V[]} values - Return an array of values
+ * @property {() => [K, V][]} entries - Return an array of entries
+ */
+
+/**
+ * @template K,V
+ * @typedef {Object} WeakStore - A safety wrapper around a WeakMap
+ * @property {(key: any) => boolean} has - Check if a key exists
+ * @property {(key: K, value: V) => void} init - Initialize the key only if it doesn't already exist
+ * @property {(key: any) => V} get - Return a value for the key. Throws
+ * if not found.
+ * @property {(key: K, value: V) => void} set - Set the key. Throws if not found.
+ * @property {(key: K) => void} delete - Remove the key. Throws if not found.
+ */
+
+/**
+ * Distinguishes between adding a new key (init) and updating or
+ * referencing a key (get, set, delete).
+ *
+ * `init` is only allowed if the key does not already exist. `Get`,
+ * `set` and `delete` are only allowed if the key does already exist.
+ * @template K,V
+ * @callback MakeWeakStore
+ * @param {string} [keyName='key'] - the column name for the key
+ * @returns {WeakStore<K,V>}
+ */
+
+/**
+ * An external store for a given constructor.
+ *
+ * @template {(...args: Array<any>) => Instance} C
+ * @typedef {Object} ExternalStore
+ * @property {C} makeInstance
+ * @property {MakeWeakStore<ReturnType<C>, any>} makeWeakStore
+ */
+
+/**
+ * @typedef {Record<string, any>} HydrateData
+ */
+
+/**
+ * @typedef {Object} HydrateHook
+ * @property {(value: any) => [string, string]} getKey
+ * @property {(key: [string, string]) => any} load
+ * @property {(storeKey: string) => void} drop
+ */
+
+/**
+ * An external store that decouples the closure data from the returned
+ * "representative" instance.
+ *
+ * @template {Array<any>} A
+ * @template {Instance} T
+ * @callback MakeHydrateExternalStore
+ * @param {string} instanceKind
+ * @param {(...args: A) => HydrateData} adaptArguments
+ * @param {(init: boolean | undefined) => (data: HydrateData) => T} makeHydrate
+ * @returns {ExternalStore<(...args: A) => T>}
+ */
+
+/**
+ * @typedef {Store<string, string> & { makeWeakStore: () => WeakStore<any, any> }}} InstanceStore
+ */
+
+/**
+ * @typedef {Object} BackingStore
+ * @property {(storeId: string, instanceKind: string) => InstanceStore} makeStore
+ * @property {(storeId: string) => InstanceStore} findStore
+ */

packages/store/src/weak-store.js

@@ -0,0 +1,40 @@
+// Copyright (C) 2019 Agoric, under Apache license 2.0
+
+// @ts-check
+
+import { assert, details, q } from '@agoric/assert';
+import './types';
+
+/**
+ * @template {Record<any, any>} K
+ * @template {any} V
+ * @param {string} [keyName='key']
+ * @returns {WeakStore<K, V>}
+ */
+export function makeWeakStore(keyName = 'key') {
+  const wm = new WeakMap();
+  const assertKeyDoesNotExist = key =>
+    assert(!wm.has(key), details`${q(keyName)} already registered: ${key}`);
+  const assertKeyExists = key =>
+    assert(wm.has(key), details`${q(keyName)} not found: ${key}`);
+  return harden({
+    has: key => wm.has(key),
+    init: (key, value) => {
+      assertKeyDoesNotExist(key);
+      wm.set(key, value);
+    },
+    get: key => {
+      assertKeyExists(key);
+      return wm.get(key);
+    },
+    set: (key, value) => {
+      assertKeyExists(key);
+      wm.set(key, value);
+    },
+    delete: key => {
+      assertKeyExists(key);
+      wm.delete(key);
+    },
+  });
+}
+harden(makeWeakStore);