Skip to content

Commit

Permalink
s3?
Browse files Browse the repository at this point in the history
  • Loading branch information
amark committed Feb 1, 2016
1 parent 3eb01bc commit f1a8801
Show file tree
Hide file tree
Showing 3 changed files with 21 additions and 67 deletions.
3 changes: 2 additions & 1 deletion lib/aws.js
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,8 @@
Bucket: s.config.bucket
,Key: key
}, id = s3.id(m);
s.on(id).once(function(e,d,t,m,r){
s.on(id).event(function(e,d,t,m,r){
this.off();
delete s.batch[id];
if(!a.fns.is(cb)){ return }
try{ cb(e,d,t,m,r);
Expand Down
83 changes: 18 additions & 65 deletions lib/s3.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
opt.s3 = opt.s3 || {};
var s3 = gun.__.opt.s3 = gun.__.opt.s3 || S3(opt && opt.s3);
s3.prefix = s3.prefix || opt.s3.prefix || '';
s3.prekey = s3.prekey || opt.s3.prekey || '';
s3.prenode = s3.prenode || opt.s3.prenode || '_/nodes/';
gun.__.opt.batch = opt.batch || gun.__.opt.batch || 10;
gun.__.opt.throttle = opt.throttle || gun.__.opt.throttle || 15;
Expand All @@ -17,49 +16,23 @@
cb = cb || function(){};
(opt = opt || {}).ctx = opt.ctx || {};
opt.ctx.load = opt.ctx.load || {};
if(key[Gun._.soul]){
key = s3.prefix + s3.prenode + Gun.is.soul(key);
} else {
key = s3.prefix + s3.prekey + key;
}
key = s3.prefix + s3.prenode + Gun.is.rel(key);
s3.GET(key, function(err, data, text, meta){
Gun.log('via s3', key, err);
if(err && err.statusCode == 404){
err = null; // we want a difference between 'unfound' (data is null) and 'error' (auth is wrong).
}
// TODO: optimize KEY command to not write data if there is only one soul (which is common).
if(meta && (meta.key || meta[Gun._.soul])){
if(err){ return cb(err) }
if(meta.key && Gun.obj.is(data) && !Gun.is.node(data)){
return Gun.obj.map(data, function(rel, soul){
if(!(soul = Gun.is.soul(rel))){ return }
opt.ctx.load[soul] = false;
s3.get(rel, cb, {next: 's3', ctx: opt.ctx}); // TODO: way faster if you use cache.
});
}
if(meta[Gun._.soul]){
return s3.get(meta, cb); // TODO: way faster if you use cache.
}
return cb({err: Gun.log('Cannot determine S3 key data!')});
}
if(data){
meta.soul = Gun.is.soul.on(data);
meta.soul = Gun.is.node.soul(data);
if(!meta.soul){
err = {err: Gun.log('No soul on node S3 data!')};
}
} else {
return cb(err, null);
}
if(err){ return cb(err) }
opt.ctx.load[meta.soul] = true;
var graph = {};
graph[meta.soul] = data;
cb(null, graph);
(graph = {})[meta.soul] = Gun.union.pseudo(meta.soul);
cb(null, graph);
if(Gun.obj.map(opt.ctx.load, function(loaded, soul){
if(!loaded){ return true }
})){ return } // return IF we have nodes still loading.
var node = data;
cb(null, node);
var node = Gun.is.node.soul.ify({}, meta.soul);
cb(null, node);
cb(null, {});
});
}
Expand All @@ -70,16 +43,19 @@
var next = s3.next
, ack = Gun.text.random(8)
, batch = s3.batch[next] = s3.batch[next] || {};
s3.on(ack).once(cb);
s3.on(ack).event(function(err, reply){
this.off();
cb(err, reply);
});
Gun.obj.map(nodes, function(node, soul){
cb.count += 1;
batch[soul] = (batch[soul] || 0) + 1;
//Gun.log("put listener for", next + ':' + soul, batch[soul], cb.count);
s3.on(next + ':' + soul).event(function(){
s3.on(next + ':' + soul).event(function(err, reply){
cb.count -= 1;
//Gun.log("transaction", cb.count);
if(!cb.count){
s3.on(ack).emit();
s3.on(ack).emit(err, reply);
this.off(); // MEMORY LEAKS EVERYWHERE!!!!!!!!!!!!!!!! FIX THIS!!!!!!!!!
}
});
Expand Down Expand Up @@ -107,7 +83,7 @@
put(exists, soul); // naive implementation of retry TODO: BUG: need backoff and anti-infinite-loop!
return;
}
s3.on(now + ':' + soul).emit(200);
s3.on(now + ':' + soul).emit(err, reply);
});
});
}
Expand All @@ -119,33 +95,10 @@
s3.persisted = s3.persisted || {};
s3.wait = s3.wait || null;

s3.key = s3.key || function(key, soul, cb){
if(!key){
return cb({err: "No key!"});
}
if(!soul){
return cb({err: "No soul!"});
}
var path = s3.prefix + s3.prekey + key, meta = {key: '0.2'}, rel = {};
meta[Gun._.soul] = rel[Gun._.soul] = soul = Gun.is.soul(soul) || soul;
s3.GET(path, function(err, data, text, _){
var souls = data || {};
souls[soul] = rel;
s3.PUT(path, souls, function(err, reply){
Gun.log("s3 key reply", soul, err, reply);
if(err || !reply){
return s3.key(key, soul, cb); // naive implementation of retry TODO: BUG: need backoff and anti-infinite-loop!
}
cb();
}, {Metadata: meta});
});
}

opt.hooks = opt.hooks || {};
gun.opt({hooks: {
get: opt.hooks.get || s3.get
,put: opt.hooks.put || s3.put
,key: opt.hooks.key || s3.key
opt.wire = opt.wire || {};
gun.opt({wire: {
get: opt.wire.get || s3.get
,put: opt.wire.put || s3.put
}}, true);
});
}());
}());
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "gun",
"version": "0.3.1",
"version": "0.3.2",
"description": "Graph engine",
"main": "index.js",
"scripts": {
Expand Down

0 comments on commit f1a8801

Please sign in to comment.