Skip to content

Commit b4584c4

Browse files
author
Miel Vander Sande
committed
Added feedback mechs
1 parent 7c5845b commit b4584c4

File tree

4 files changed

+195
-42
lines changed

4 files changed

+195
-42
lines changed

‎lib/controllers/SummaryController.js

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ function SummaryController(options) {
1818
var summaries = options.summaries || {};
1919
this._summariesFolder = path.join(__dirname, summaries.dir || '../../summaries');
2020

21+
console.log('The summary folder is ' + this._summariesFolder);
22+
2123
// Set up path matching
2224
this._summariesPath = summaries.path || '/summaries/',
2325
this._matcher = new RegExp('^' + Util.toRegExp(this._summariesPath) + '(.+)$');

‎lib/datasources/SummaryDatasource.js

Lines changed: 81 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,13 @@ var Datasource = require('./Datasource'),
22
N3 = require('n3'),
33
path = require('path'),
44
fs = require('fs'),
5-
Bloem = require('bloem').Bloem;
5+
Bloem = require('bloem').Bloem,
6+
chokidar = require('chokidar');
67

78
var ACCEPT = 'text/turtle;q=1.0,application/n-triples;q=0.7,text/n3;q=0.6';
89

9-
var DCT = 'http://purl.org/dc/terms/',
10+
var RDF = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#',
11+
DCT = 'http://purl.org/dc/terms/',
1012
AMF = 'http://semweb.mmlab.be/ns/membership#',
1113
DS = 'http://semweb.mmlab.be/ns/summaries#';
1214

@@ -17,66 +19,103 @@ function SummaryDatasource(options) {
1719
Datasource.call(this, options);
1820

1921
// Settings for data summaries
20-
this._summariesFolder = options.dir || '../../summaries';
22+
this._summariesFolder = (options.dir && path.isAbsolute(options.dir)) ? options.dir : path.join(__dirname, options.dir || '../../summaries');
2123
}
2224
Datasource.extend(SummaryDatasource, ['triplePattern', 'limit', 'offset']);
2325

2426
// Prepares the datasource for querying
2527
SummaryDatasource.prototype._initialize = function (done) {
26-
var tripleStore = this._tripleStore = new N3.Store();
28+
this._tripleStore = new N3.Store();
29+
// var parser = new N3.Parser();
30+
// var self = this;
31+
32+
// If summaryDir does not exist, create it
33+
if (!fs.existsSync(this._summariesFolder)) {
34+
fs.mkdirSync(this._summariesFolder, { recursive: true });
35+
}
36+
37+
// Initialize watcher.
38+
const watcher = chokidar.watch(this._summariesFolder, {
39+
ignored: /(^|[\/\\])\../,
40+
persistent: true
41+
});
42+
43+
console.log(`Watching ${this._summariesFolder}`)
44+
watcher.on('add', (summaryFile) => this._storeFile(summaryFile, err => console.log(err)));
45+
done();
46+
};
47+
48+
SummaryDatasource.prototype._storeFile = function (summaryFile, callback) {
49+
console.log(`Adding ${summaryFile} to store`)
2750
var parser = new N3.Parser();
51+
52+
var document = this._fetch({ url: summaryFile, headers: { accept: ACCEPT } }, callback);
53+
var tripleStore = this._tripleStore;
2854
var self = this;
55+
// N3Parser._resetBlankNodeIds();
2956

30-
fs.readdir(this._summariesFolder, function (err, items) {
31-
if (err) {
32-
done(err);
57+
var graph = decodeURIComponent(summaryFile);
58+
parser.parse(document, function (error, triple) {
59+
if (error) {
60+
callback && callback(error);
3361
return;
3462
}
35-
36-
// Store every summary in the store
37-
items.forEach(function (summaryFile) {
38-
var document = self._fetch({ url: path.join(self._summariesFolder, summaryFile), headers: { accept: ACCEPT } }, done);
39-
// N3Parser._resetBlankNodeIds();
40-
parser.parse(document, function (error, triple) {
41-
if (error) {
42-
done(error);
43-
return;
44-
}
45-
46-
if (triple)
47-
tripleStore.addTriple(triple.subject, triple.predicate, triple.object, summaryFile);
48-
else {
49-
// Create AMFs
50-
self._storeFilters();
51-
done();
52-
}
53-
});
54-
});
63+
64+
if (triple)
65+
tripleStore.addTriple(triple.subject, triple.predicate, triple.object, graph);
66+
else {
67+
tripleStore.addTriple(graph, RDF + 'type' , DS + 'Summary', graph);
68+
// Create AMFs
69+
console.log(`Storing filters of ${graph}`)
70+
self._storeFilter(graph);
71+
callback && callback(null);
72+
}
5573
});
56-
};
74+
}
5775

58-
SummaryDatasource.prototype._storeFilters = function () {
76+
SummaryDatasource.prototype._storeFilter = function (graph) {
5977
var filters = this._filters = {};
6078
var tripleStore = this._tripleStore;
6179

62-
var filterQuads = tripleStore.find(null, DS + 'objFilter', null);
63-
80+
var filterQuads = tripleStore.find(null, DS + 'objFilter', null, graph);
81+
console.log(`Found ${filterQuads.length} filters for ${graph}`)
6482
filterQuads.forEach(function (quad) {
65-
var bitsString = tripleStore.find(quad.object, AMF + 'bits', null)[0].object;
66-
var bits = parseInt(N3.Util.getLiteralValue(bitsString), 10);
83+
try {
84+
var bitsMatches = tripleStore.find(quad.object, AMF + 'bits', null, graph);
85+
86+
if (bitsMatches.length === 0) return;
87+
88+
var bitsString = bitsMatches[0].object;
89+
var bits = parseInt(N3.Util.getLiteralValue(bitsString), 10);
90+
91+
var hashesMatches = tripleStore.find(quad.object, AMF + 'hashes', null, graph);
92+
93+
if (hashesMatches.length === 0) return;
6794

68-
var hashesString = tripleStore.find(quad.object, AMF + 'hashes', null)[0].object;
69-
var hashes = parseInt(N3.Util.getLiteralValue(hashesString), 10);
95+
var hashesString = hashesMatches[0].object;
96+
var hashes = parseInt(N3.Util.getLiteralValue(hashesString), 10);
7097

71-
// Decode filter
72-
var filterString = tripleStore.find(quad.object, AMF + 'filter', null)[0].object;
73-
var filter = Buffer.from(N3.Util.getLiteralValue(filterString), 'base64');
98+
// Decode filter
99+
var filterMatches = tripleStore.find(quad.object, AMF + 'filter', null, graph);
74100

75-
// Find predicate
76-
var predicate = tripleStore.find(quad.subject, DS + 'predicate', null, quad.graph)[0].object;
101+
if (filterMatches.length === 0) return;
77102

78-
filters[predicate] = filters[predicate] || {}; // create entry for predicate if not exists
79-
filters[predicate][quad.graph] = new Bloem(bits, hashes, filter); // add filter for the summary
103+
var filterString = filterMatches[0].object;
104+
var filter = Buffer.from(N3.Util.getLiteralValue(filterString), 'base64');
105+
106+
// Find predicate
107+
var predicateMatches = tripleStore.find(quad.subject, DS + 'predicate', null, quad.graph);
108+
109+
if (predicateMatches.length === 0) return;
110+
111+
var predicate = predicateMatches[0].object;
112+
113+
filters[predicate] = filters[predicate] || {}; // create entry for predicate if not exists
114+
filters[predicate][quad.graph] = new Bloem(bits, hashes, filter); // add filter for the summary
115+
} catch (er) {
116+
console.log(er)
117+
console.log(filterMatches)
118+
}
80119
});
81120
};
82121

‎package-lock.json

Lines changed: 111 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
"dependencies": {
2525
"asynciterator": "^1.1.0",
2626
"bloem": "^0.2.4",
27+
"chokidar": "^3.0.2",
2728
"forwarded-parse": "^2.0.0",
2829
"jsonld": "^0.4.11",
2930
"lodash": "^2.4.2",

0 commit comments

Comments
 (0)