Optimize (#6)

* Replaced validate(key) === type with assert(key, type). Disabled upstash SDK telemetry

* Comments/docs in api/helper.js

* Memory optimization: store only hash part of key as redis-key

Author: SomajitDey

api/helper.js

@@ -76,6 +76,7 @@ export function id(){
     return sign('id');
 }
 
+// Check if given key is valid, and if it is, return its type
 export function validate(key, silent=true){
     const sig = key.substring(0, sigLen);
     const hash = key.substring(sigLen);
@@ -89,18 +90,33 @@ export function validate(key, silent=true){
     }
 }
 
+// Assert if given key is of the given type (private | public)
+// This is computationally favorable to the Boolean: validate(key) === type
+export function assert(key, type){
+    const sig = key.substring(0, sigLen);
+    const hash = key.substring(sigLen);
+    return sig === sign(hash + type);
+}
+
 export function genPublicKey(privateOrPublicKey){
-    if (validate(privateOrPublicKey) === 'public') return privateOrPublicKey;
+    if (assert(privateOrPublicKey, 'public')) return privateOrPublicKey;
     const privateKey = privateOrPublicKey;
     const privateHash = privateKey.substring(sigLen);
     const publicHash = hash(privateHash);
     const publicKey = sign(publicHash + 'public') + publicHash;
     return publicKey;
 }
 
+export function genKeyPair(seed = randomUUID()){
+    const privateHash = hash(seed);
+    const privateKey = sign(privateHash + 'private') + privateHash;
+    const publicKey = genPublicKey(privateKey);
+    return {private: privateKey, public: publicKey};
+}
+
 export function cacheSet(privateKey, obj){
     const publicKey = genPublicKey(privateKey);
-    const dbKey = dbKeyPrefix.cache + publicKey;
+    const dbKey = dbKeyPrefix.cache + publicKey.substring(sigLen);
     // Promise.all below enables both commands to be executed in a single http request (using same pipeline)
     // As Redis is single-threaded, the commands are executed in order
     // See https://upstash.com/docs/redis/sdks/ts/pipelining/auto-pipeline
@@ -111,23 +127,16 @@ export function cacheSet(privateKey, obj){
 }
 
 export function cacheGet(publicKey, key){
-    const dbKey = dbKeyPrefix.cache + publicKey;
+    const dbKey = dbKeyPrefix.cache + publicKey.substring(sigLen);
     return redisRateLimit.hget(dbKey, key);
 }
 
 export function cacheDel(privateOrPublicKey, key){
     const publicKey = genPublicKey(privateOrPublicKey);
-    const dbKey = dbKeyPrefix.cache + publicKey;
+    const dbKey = dbKeyPrefix.cache + publicKey.substring(sigLen);
     return redisRateLimit.hdel(dbKey, key);
 }
 
-export function genKeyPair(seed = randomUUID()){
-    const privateHash = hash(seed);
-    const privateKey = sign(privateHash + 'private') + privateHash;
-    const publicKey = genPublicKey(privateKey);
-    return {private: privateKey, public: publicKey};
-}
-
 // Add metadata to payload (which must be a JSON object)
 // Some metadata, such as `id` to uniquely identify a payload and timestamp, are generated
 // Other metadata may be provided as the `fields` JSON object.
@@ -151,7 +160,7 @@ export function unlockJSON(json, passwd){
 }
 
 export async function publicProduce(publicKey, data){
-    const dbKey = dbKeyPrefix.manyToOne + publicKey;
+    const dbKey = dbKeyPrefix.manyToOne + publicKey.substring(sigLen);
     return Promise.all([
       redisData.rpush(dbKey, data),
       redisData.expire(dbKey, ttl)
@@ -160,7 +169,7 @@ export async function publicProduce(publicKey, data){
 
 export async function privateConsume(privateKey){
     const publicKey = genPublicKey(privateKey);
-    const dbKey = dbKeyPrefix.manyToOne + publicKey;
+    const dbKey = dbKeyPrefix.manyToOne + publicKey.substring(sigLen);
     const atomicTransaction = redisData.multi();
     atomicTransaction.lrange(dbKey, 0, -1);
     atomicTransaction.del(dbKey);
@@ -170,26 +179,26 @@ export async function privateConsume(privateKey){
 
 export async function privateProduce(privateKey, data){
     const publicKey = genPublicKey(privateKey);
-    const dbKey = dbKeyPrefix.oneToMany + publicKey;
+    const dbKey = dbKeyPrefix.oneToMany + publicKey.substring(sigLen);
     return redisData.set(dbKey, data, { ex: ttl });
 }
 
 export async function privateDelete(privateKey){
     const publicKey = genPublicKey(privateKey);
-    const dbKey = dbKeyPrefix.oneToMany + publicKey;
+    const dbKey = dbKeyPrefix.oneToMany + publicKey.substring(sigLen);
     return redisData.del(dbKey);
 }
 
 export async function privateRefresh(privateKey){
     const publicKey = genPublicKey(privateKey);
-    const dbKey = dbKeyPrefix.oneToMany + publicKey;
+    const dbKey = dbKeyPrefix.oneToMany + publicKey.substring(sigLen);
     return redisData.expire(dbKey, ttl);
 }
 
 export async function privateStats(privateKey){
     const publicKey = genPublicKey(privateKey);
-    const dbKeyConsume = dbKeyPrefix.manyToOne + publicKey;
-    const dbKeyPublish = dbKeyPrefix.oneToMany + publicKey;
+    const dbKeyConsume = dbKeyPrefix.manyToOne + publicKey.substring(sigLen);
+    const dbKeyPublish = dbKeyPrefix.oneToMany + publicKey.substring(sigLen);
     const [ countConsume, ttlConsume ] = await Promise.all([
       redisData.llen(dbKeyConsume),
       redisData.ttl(dbKeyConsume)
@@ -202,7 +211,7 @@ export async function privateStats(privateKey){
 
 // Demand for data also refreshes its expiry
 export async function publicConsume(publicKey){
-    const dbKey = dbKeyPrefix.oneToMany + publicKey;
+    const dbKey = dbKeyPrefix.oneToMany + publicKey.substring(sigLen);
     // Ideally there should be getex() in Upstash's Redis SDK.
     // Until it's available, we make do with pipelining as follows.
     const [ data, _ ] = await Promise.all([
@@ -214,16 +223,18 @@ export async function publicConsume(publicKey){
 
 export async function oneToOneProduce(privateKey, key, data){
     const publicKey = genPublicKey(privateKey);
-    const dbKey = dbKeyPrefix.oneToOne + publicKey;
+    const dbKey = dbKeyPrefix.oneToOne + publicKey.substring(sigLen);
     const field = {[hash(key)]: data};
+    // Ideally there should be hexpire() in Upstash's Redis SDK.
+    // Until it's available, we expire the containing key as follows.
     return Promise.all([
       redisData.hset(dbKey, field),
       redisData.expire(dbKey, ttl)
     ])
 }
 
 export async function oneToOneConsume(publicKey, key){
-    const dbKey = dbKeyPrefix.oneToOne + publicKey;
+    const dbKey = dbKeyPrefix.oneToOne + publicKey.substring(sigLen);
     const field = hash(key);
     const atomicTransaction = redisData.multi();
     atomicTransaction.hget(dbKey, field);
@@ -234,8 +245,10 @@ export async function oneToOneConsume(publicKey, key){
 
 export async function oneToOneTTL(privateKey, key){
     const publicKey = genPublicKey(privateKey);
-    const dbKey = dbKeyPrefix.oneToOne + publicKey;
+    const dbKey = dbKeyPrefix.oneToOne + publicKey.substring(sigLen);
     const field = hash(key);
+    // Ideally there should be httl() in Upstash's Redis SDK.
+    // Until it's available, we use ttl of the containing key as follows.
     const [ bool, ttl ] = await Promise.all([
       redisData.hexists(dbKey, field),
       redisData.ttl(dbKey)
@@ -244,13 +257,14 @@ export async function oneToOneTTL(privateKey, key){
 }
 
 // Tokens are stored in LIFO stacks. Old and unused tokens are trimmed.
+// Timestamps are stored with the tokens using string concatenation
 export async function streamToken(privateOrPublicKey, receive=true){
   const type = validate(privateOrPublicKey, false);
   const typeComplement = (type == 'private') ? 'public' : 'private';
   const publicKey = genPublicKey(privateOrPublicKey);
   const mode = receive ? "receive" : "send";
   const modeComplement = receive ? "send" : "receive";
-  const existing = await redisData.lpop(dbKeyPrefix.stream[typeComplement][modeComplement] + publicKey);
+  const existing = await redisData.lpop(dbKeyPrefix.stream[typeComplement][modeComplement] + publicKey.substring(sigLen));
   const timeNow = Math.round(Date.now()/1000);
   if (existing) {
     const [token, timestamp] = existing.split('@');
@@ -259,7 +273,7 @@ export async function streamToken(privateOrPublicKey, receive=true){
     if ((timeNow - timestamp) < streamTimeout) return token;
   }
   const token = randStr();
-  const dbKey = dbKeyPrefix.stream[type][mode] + publicKey;
+  const dbKey = dbKeyPrefix.stream[type][mode] + publicKey.substring(sigLen);
   await Promise.all([
     redisData.lpush(dbKey, token + '@' + timeNow),
     redisData.expire(dbKey, streamTimeout),

api/index.js

@@ -83,7 +83,7 @@ fastify.post('/public/:publicKey', async (request, reply) => {
     const redirectOnErr = request.query.err;
 
     try {
-        if (helper.validate(publicKey) !== 'public') throw 401;
+        if (!helper.assert(publicKey, 'public')) throw 401;
 
         const app = request.query.app;
         const data = helper.decoratePayload(request.body);
@@ -142,7 +142,7 @@ fastify.get('/private/:privateKey', async (request, reply) => {
     webhookHandler = statsHandler = dataHandler = () => null; // default
 
     try {
-        if (helper.validate(privateKey) !== 'private') throw 401;
+        if (!helper.assert(privateKey, 'private')) throw 401;
         if (webhook == null) {
             webhookHandler = () => helper.cacheDel(privateKey, 'hook');
         } else {
@@ -180,7 +180,7 @@ fastify.post('/private/:privateKey', async (request, reply) => {
     const redirectOnErr = request.query.err;
     const passwd = request.query.password;
     try {
-        if (helper.validate(privateKey) !== 'private') throw 401;
+        if (!helper.assert(privateKey, 'private')) throw 401;
 
         const cdn = {};
         if (passwd == null) {
@@ -218,7 +218,7 @@ fastify.delete('/private/:privateKey', async (request, reply) => {
     const { privateKey } = request.params;
     const passwdPresent = 'password' in request.query;
     try {
-        if (helper.validate(privateKey) !== 'private') throw 401;
+        if (!helper.assert(privateKey, 'private')) throw 401;
         if (passwdPresent) {
           await helper.privateDelete(privateKey);
         } else {
@@ -238,7 +238,7 @@ fastify.patch('/private/:privateKey', async (request, reply) => {
     const { privateKey } = request.params;
     const passwdPresent = 'password' in request.query;
     try {
-        if (helper.validate(privateKey) !== 'private') throw 401;
+        if (!helper.assert(privateKey, 'private')) throw 401;
 
         const cdn = {};
         if (passwdPresent) {
@@ -267,7 +267,7 @@ fastify.get('/public/:publicKey', async (request, reply) => {
     const { publicKey } = request.params;
     const passwd = request.query.password;
     try {
-        if (helper.validate(publicKey) !== 'public') throw 401;
+        if (!helper.assert(publicKey, 'public')) throw 401;
 
         if (passwd == null) {
           reply.redirect(`${cdnUrlBase}/${publicKey}.json`, 301);
@@ -297,7 +297,7 @@ fastify.post('/private/:privateKey/:key', async (request, reply) => {
     const redirectOnOk = request.query.ok;
     const redirectOnErr = request.query.err;
     try {
-        if (helper.validate(privateKey) !== 'private') throw 401;
+        if (!helper.assert(privateKey, 'private')) throw 401;
         await helper.oneToOneProduce(privateKey, key, JSON.stringify(helper.decoratePayload(request.body)));
         if (redirectOnOk == null) {
             reply.send({message: "Done", error: "Ok", statusCode: reply.statusCode});
@@ -322,7 +322,7 @@ fastify.post('/private/:privateKey/:key', async (request, reply) => {
 fastify.get('/public/:publicKey/:key', async (request, reply) => {
     const { publicKey, key } = request.params;
     try {
-        if (helper.validate(publicKey) !== 'public') throw 401;
+        if (!helper.assert(publicKey, 'public')) throw 401;
         const data = await helper.oneToOneConsume(publicKey, key);
         if (!data) throw 404;
         reply.send(data);
@@ -342,7 +342,7 @@ fastify.get('/public/:publicKey/:key', async (request, reply) => {
 fastify.get('/private/:privateKey/:key', async (request, reply) => {
     const { privateKey, key } = request.params;
     try {
-        if (helper.validate(privateKey) !== 'private') throw 401;
+        if (!helper.assert(privateKey, 'private')) throw 401;
         return helper.oneToOneTTL(privateKey, key);
     } catch (err) {
         if (err == 400) {

api/test.js

@@ -14,9 +14,9 @@ const key = helper.genKeyPair();
 
 console.log(JSON.stringify(key));
 
-console.log('This should show public: ' + helper.validate(key.public));
-console.log('This should show private: ' + helper.validate(key.private));
-console.log('This should show false: ' + helper.validate('random'));
+console.log('This should show public: ', helper.validate(key.public), helper.assert(key.public, 'public'));
+console.log('This should show private: ', helper.validate(key.private), helper.assert(key.private, 'private'));
+console.log('This should show false: ', helper.validate('random'), helper.assert('random', 'private') || helper.assert('random', 'public'));
 
 console.log('Stream token private POST:', await helper.streamToken(key.private, false));
 console.log('Stream token public GET:', await helper.streamToken(key.public));

example.env

@@ -23,6 +23,8 @@ KV_REST_API_TOKEN=
 # Redis credentials to be used for database
 UPSTASH_REDIS_REST_URL=
 UPSTASH_REDIS_REST_TOKEN=
+# Disable upstash SDK telemetry for performance
+UPSTASH_DISABLE_TELEMETRY=1
 
 # OneSignal credentials for web-push for each app registered @ https://github.com/securelay/apps
 ONESIGNAL_API_KEY_FORMONIT=