Skip to content

Aggregator file for browser plugin #6258

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
May 28, 2025
Merged

Aggregator file for browser plugin #6258

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
75cf20c
Aggregator file for browser plugin
May 26, 2025
4ecf68b
nullcheck
May 26, 2025
cabe892
Merge branch 'newarchitecture' of https://github.com/Countly/countly-…
May 26, 2025
f99b9ac
fixes
May 27, 2025
bfe266e
fixes for tests
May 27, 2025
fb7f030
fixes for tests
May 27, 2025
ea30f06
fx
May 27, 2025
45c26af
test fixes
May 27, 2025
9e01e2f
not caching test results
May 27, 2025
89f6f3c
fx
May 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion api/parts/jobs/handle.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@ class Handle {
* @returns {Job} job
**/
job(name, data) {
let Constructor = this.classes[name];

let Constructor = this.classes && this.classes[name];
if (Constructor) {
return new Constructor(name, data);
}
Expand Down
6 changes: 6 additions & 0 deletions api/utils/calculatedDataManager.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@ const log = require('./log.js')('core:calculatedDataManager');
* @param {object} options - options object
* @param {object} options.query_data - query data
* @param {object} options.db - db connection
* @param {string} options.id - id of the query, if not given it will be calculated based on query_data
* @param {boolean} [options.no_cache=false] - if true, will not use cache
* @param {boolean} [options.returned=false] - if true, will not output data
* @param {function} options.outputData - function to output data
* @param {number} options.threshold - threshold in seconds
*/
Expand All @@ -32,6 +35,9 @@ calculatedDataManager.longtask = async function(options) {
var timeout;
var keep = parseInt(plugins.getConfig("drill").drill_snapshots_cache_time, 10) || 60 * 60 * 24;
keep = keep * 1000;
if (options.no_cache) {
keep = 0;
}

/**
* Return message in case it takes too long
Expand Down
3 changes: 2 additions & 1 deletion api/utils/requestProcessor.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2772,11 +2772,12 @@ const processRequest = (params) => {
params.qstring.query.period = params.qstring.query.period || params.qstring.period || "30days";
}
if (params.qstring.periodOffset) {
params.qstring.query.periodOffset = params.qstrig.query.periodOffset || params.qstring.periodOffset || 0;
params.qstring.query.periodOffset = params.qstring.query.periodOffset || params.qstring.periodOffset || 0;
}

calculatedDataManager.longtask({
db: common.db,
no_cache: params.qstring.no_cache,
threshold: plugins.getConfig("api").request_threshold,
app_id: params.qstring.query.app_id,
query_data: params.qstring.query,
Expand Down
14 changes: 14 additions & 0 deletions plugins/browser/api/aggregator.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@

var plugins = require('../../pluginManager.js');

(function() {
plugins.register("/session/metrics", function(ob) {
ob.predefinedMetrics.push({
db: "browser",
metrics: [
{ name: "_browser", set: "browser", short_code: "brw" },
{ name: "_browser_version", set: "browser_version", short_code: "brwv" }
]
});
});
}());
15 changes: 14 additions & 1 deletion plugins/star-rating/tests.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -424,7 +424,20 @@ describe('Testing Rating plugin', function() {
}
var ob = JSON.parse(res.text);
ob.should.have.property('result', 'Success');
setTimeout(done, 10 * testUtils.testScalingFactor);
//giving morre time to remove widget
setTimeout(done, 10 * testUtils.testScalingFactor + 2000);
});
});
it('should not find widget in database', function(done) {
request.get('/o/feedback/widget?app_id=' + APP_ID + '&api_key=' + API_KEY_ADMIN + '&widget_id=' + WIDGET_ID)
.expect(404)
.end(function(err, res) {
if (err) {
return done(err);
}
var ob = JSON.parse(res.text);
ob.should.have.property('result', 'Widget not found.');
done();
});
});
});
Expand Down
16 changes: 8 additions & 8 deletions plugins/views/tests.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ function verifySegments(values) {
function verifyTotals(period, order, orderString) {
it("Checking against calculating same table from granural data", function(done) {
request
.get('/o/aggregate?api_key=' + API_KEY_ADMIN + '&app_id=' + APP_ID + '&query={"queryName":"viewsTableData"}&period=' + period)
.get('/o/aggregate?api_key=' + API_KEY_ADMIN + '&no_cache=true&app_id=' + APP_ID + '&query={"queryName":"viewsTableData"}&period=' + period)
.expect(200)
.end(function(err, res) {
console.log(res.text);
Expand Down Expand Up @@ -341,12 +341,12 @@ describe('Testing views plugin', function() {
.get('/i?app_key=' + APP_KEY + '&device_id=' + "user1" + '&timestamp=' + (myTime - (25 * 24 * 60 * 60 * 1000)) + '&events=' + data)
.expect(200)
.end(function(err, res) {
setTimeout(done, 1000 * testUtils.testScalingFactor);
setTimeout(done, 1000 * testUtils.testScalingFactor + 2000);
});
});
});

describe('verifying totals after last update', function() {
describe('verifying totals after last update(last year+ 25 days ago)', function() {
verifyTotals("30days");
verifyTotals("month");
});
Expand Down Expand Up @@ -383,7 +383,7 @@ describe('Testing views plugin', function() {
});
});
});
describe('verifying totals after last update', function() {
describe('verifying totals after last update(yesterday)', function() {
verifyTotals("yesterday");
verifyTotals("30days");
verifyTotals("month");
Expand All @@ -407,11 +407,11 @@ describe('Testing views plugin', function() {
.get('/i?app_key=' + APP_KEY + '&device_id=' + "user1" + '&timestamp=' + (myTime - 10) + '&events=' + data)
.expect(200)
.end(function(err, res) {
setTimeout(done, 1000 * testUtils.testScalingFactor);
setTimeout(done, 1000 * testUtils.testScalingFactor + 2000);
});
});
});
describe('verifying totals after last update', function() {
describe('verifying totals after last update(right now)', function() {
verifyTotals("hour");
verifyTotals("yesterday");
verifyTotals("30days");
Expand All @@ -437,11 +437,11 @@ describe('Testing views plugin', function() {
.get('/i?app_key=' + APP_KEY + '&device_id=' + "user1" + '&timestamp=' + (myTime) + '&events=' + data)
.expect(200)
.end(function(err, res) {
setTimeout(done, 1000 * testUtils.testScalingFactor);
setTimeout(done, 1000 * testUtils.testScalingFactor + 2000);
});
});
});
describe('verifying totals after last update', function() {
describe('verifying totals after last update(Same user different view)', function() {
verifyTotals("hour");
verifyTotals("yesterday");
verifyTotals("30days");
Expand Down
2 changes: 1 addition & 1 deletion test/3.api.write/2.write.app.sessions.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ var DEVICE_ID = "1234567890";

var compareAgainstGranuralData = function(options, data, done) {
request
.get('/o/aggregate?api_key=' + API_KEY_ADMIN + '&app_id=' + APP_ID + '&query=' + JSON.stringify(options.query) + '&period=' + (options.period || "30ddays"))
.get('/o/aggregate?api_key=' + API_KEY_ADMIN + '&no_cache=true&app_id=' + APP_ID + '&query=' + JSON.stringify(options.query) + '&period=' + (options.period || "30ddays"))
.expect(200)
.end(function(err, res) {
if (err) {
Expand Down
15 changes: 8 additions & 7 deletions test/3.api.write/4.write.app.metrics.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ var DEVICE_ID = "1234567890";

var compareAgainstGranuralData = function(options, data, done) {
request
.get('/o/aggregate?api_key=' + API_KEY_ADMIN + '&app_id=' + APP_ID + '&query=' + JSON.stringify(options.query) + '&period=' + (options.period || "30ddays"))
.get('/o/aggregate?api_key=' + API_KEY_ADMIN + '&no_cache=true&app_id=' + APP_ID + '&query=' + JSON.stringify(options.query) + '&period=' + (options.period || "30ddays"))
.expect(200)
.end(function(err, res) {
if (err) {
Expand Down Expand Up @@ -185,7 +185,7 @@ describe('Writing app metrics', function() {
}
var ob = JSON.parse(res.text);
ob.should.have.property('result', 'Success');
setTimeout(done, 1000 * testUtils.testScalingFactor);
setTimeout(done, 1000 * testUtils.testScalingFactor + 2000);
});
});
});
Expand All @@ -196,6 +196,7 @@ describe('Writing app metrics', function() {
.get('/o?api_key=' + API_KEY_ADMIN + '&app_id=' + APP_ID + '&method=device_details')
.expect(200)
.end(function(err, res) {
console.log(JSON.stringify(res));
testUtils.validateMetrics(err, res, done, {meta: {"os": ["Android"], "os_versions": ["a4:4"]}, Android: {"n": 2, "t": 2, "u": 2}, "a4:4": {"n": 1, "t": 1, "u": 1}});
});
});
Expand All @@ -221,7 +222,7 @@ describe('Writing app metrics', function() {
}
var ob = JSON.parse(res.text);
ob.should.have.property('result', 'Success');
setTimeout(done, 1000 * testUtils.testScalingFactor);
setTimeout(done, 1000 * testUtils.testScalingFactor + 2000);
});
});
});
Expand Down Expand Up @@ -270,7 +271,7 @@ describe('Writing app metrics', function() {
}
var ob = JSON.parse(res.text);
ob.should.have.property('result', 'Success');
setTimeout(done, 1000 * testUtils.testScalingFactor);
setTimeout(done, 1000 * testUtils.testScalingFactor + 2000);
});
});
});
Expand Down Expand Up @@ -359,7 +360,7 @@ describe('Writing app metrics', function() {
}
var ob = JSON.parse(res.text);
ob.should.have.property('result', 'Success');
setTimeout(done, 1000 * testUtils.testScalingFactor);
setTimeout(done, 1000 * testUtils.testScalingFactor + 2000);
});
});
});
Expand Down Expand Up @@ -391,7 +392,7 @@ describe('Writing app metrics', function() {
}
var ob = JSON.parse(res.text);
ob.should.have.property('result', 'Success');
setTimeout(done, 1000 * testUtils.testScalingFactor);
setTimeout(done, 1000 * testUtils.testScalingFactor + 2000);
});
});
});
Expand Down Expand Up @@ -478,7 +479,7 @@ describe('Writing app metrics', function() {
}
var ob = JSON.parse(res.text);
ob.should.have.property('result', 'Success');
setTimeout(done, 1000 * testUtils.testScalingFactor);
setTimeout(done, 1000 * testUtils.testScalingFactor + 2000);
});
});
});
Expand Down
34 changes: 32 additions & 2 deletions test/3.api.write/5.write.app.events.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ var APP_KEY = "";
var API_KEY_ADMIN = "";
var APP_ID = "";
var DEVICE_ID = "1234567890";
var drill_db = "";

var totalEventCounts = {};

Expand Down Expand Up @@ -52,6 +53,7 @@ describe('Writing app events', function() {
API_KEY_ADMIN = testUtils.get("API_KEY_ADMIN");
APP_ID = testUtils.get("APP_ID");
APP_KEY = testUtils.get("APP_KEY");
drill_db = testUtils.client.db("countly_drill");
var params = [];
request
.get('/i?device_id=' + DEVICE_ID + '&app_key=' + APP_KEY + "&events=" + JSON.stringify(params))
Expand Down Expand Up @@ -167,8 +169,15 @@ describe('Writing app events', function() {
});
});
/*{"2015":{"1":{"6":{"17":{"c":1},"c":1},"c":1},"c":1}}
*/
*/
describe('verify events without params', function() {
it("Verify granural data", function(done) {
testUtils.validateTotalsInDrillData(
drill_db,
{app_id: APP_ID, event: "test", query: {}, values: {u: 1, t: 2}},
done
);
});
it('should have 2 event', function(done) {
request
.get('/o?api_key=' + API_KEY_ADMIN + '&app_id=' + APP_ID + '&method=events')
Expand Down Expand Up @@ -256,6 +265,13 @@ describe('Writing app events', function() {
});
});
describe('verify events without params', function() {
it("Verify granural data", function(done) {
testUtils.validateTotalsInDrillData(
drill_db,
{app_id: APP_ID, event: "test", query: {}, values: {u: 1, t: 4}},
done
);
});
it('should have 4 test events', function(done) {
request
.get('/o?api_key=' + API_KEY_ADMIN + '&app_id=' + APP_ID + '&method=events')
Expand Down Expand Up @@ -340,12 +356,26 @@ describe('Writing app events', function() {
});
});
describe('verify events without params', function() {
it("Verify granural data(test)", function(done) {
testUtils.validateTotalsInDrillData(
drill_db,
{app_id: APP_ID, event: "test", query: {}, values: {u: 1, t: 4}},
done
);
});
it("Verify granural data(test1)", function(done) {
testUtils.validateTotalsInDrillData(
drill_db,
{app_id: APP_ID, event: "test1", query: {}, values: {u: 1, t: 1}},
done
);
});
it('should 1 test1 and 4 test events', function(done) {
request
.get('/o?api_key=' + API_KEY_ADMIN + '&app_id=' + APP_ID + '&method=events')
.expect(200)
.end(function(err, res) {
testUtils.validateEvents(err, res, done, {c: 4});
testUtils.validateEvents(err, res, done, {c: 5});
});
});
});
Expand Down
2 changes: 1 addition & 1 deletion test/3.api.write/8.checksums.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ describe("Testing checksum validations", function() {

const responseObject = JSON.parse(response.text);
responseObject.should.have.property("salt", TEST_SALT);
setTimeout(done, 1000 * testUtils.testScalingFactor);
setTimeout(done, 1000 * testUtils.testScalingFactor + 1000);
});
});
});
Expand Down
2 changes: 1 addition & 1 deletion test/testUtils.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -287,7 +287,7 @@ var testUtils = function testUtils() {

var pipeline = [{"$match": match}];
options.values = options.values || {};
pipeline.push({"$group": {"_id": "$uid", "t": {"$sum": 1}, "ls": {"$min": "$up.ls"}, "fs": {"$min": "$up.fs"}}});
pipeline.push({"$group": {"_id": "$uid", "t": {"$sum": "$c"}, "ls": {"$min": "$up.ls"}, "fs": {"$min": "$up.fs"}}});
pipeline.push({"$group": {"_id": null, "u": {"$sum": 1}, "t": {"$sum": "$t"}, "fs": {"$min": "$fs"}, "ls": {"$min": "$ls"}}});
console.log(JSON.stringify(pipeline));
db.collection("drill_events").aggregate(pipeline, function(err, res) {
Expand Down
46 changes: 27 additions & 19 deletions test/unit-tests/db.tests.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -256,41 +256,49 @@ describe('Testing Simple database operations', function() {
});

describe("test insert Many(+behaviour on duplicate)", function() {
it("Insert once and check result(promise)", async function(done) {
var rr = await db.collection("testCommands3").insertMany([{"_id": 1}, { "_id": 2}, {"_id": 3}]);
it("Insert once and check result(promise)", async function() {
try {
var rr = await db.collection("testCommands3").insertMany([{"_id": 1}, { "_id": 2}, {"_id": 3}]);
}
catch (err) {
console.log("Error: " + JSON.stringify(err));
}

var cursor = db.collection("testCommands3").find();
var res = await cursor.toArray();
res.should.have.property.length(3);
res.should.have.property("length", 3);
});

it("Insert again and check result(callback)", async function(done) {
await db.collection("testCommands3").insertMany([{"_id": 4}, { "_id": 5}, { "_id": 6}]);
it("Insert again and check result(callback)", function(done) {
db.collection("testCommands3").insertMany([{"_id": 4}, { "_id": 5}, { "_id": 6}], function(err, res) {
db.collection("testCommands3").find().toArray(function(err, res) {
if (!res || res.length !== 6) {
done("Error: " + JSON.stringify(res));
}
else {
done();
}

db.collection("testCommands3").insertMany([{"_id": 4}, { "_id": 5}, { "_id": 6}], function(err, res) {
db.collection("testCommands3").find().toArray(function(err, res) {
if (!res || res.length !== 6) {
done("Error: " + JSON.stringify(res));
}
else {
done();
}
});
});
});
});

it("Insert and should get duplicate error", async function(done) {
var rr = await db.collection("testCommands3").insertMany([{"_id": 7}, { "_id": 1}, { "_id": 8}]);
it("Insert and should get duplicate error", async function() {
try {
var rr = await db.collection("testCommands3").insertMany([{"_id": 7}, { "_id": 1}, { "_id": 8}], {"ordered": false});
}
catch (err) {
console.log("Error: " + JSON.stringify(err));
}

var cursor = db.collection("testCommands3").find();
var res = await cursor.toArray();
if (!res || res.length !== 8) {
done("Error: " + JSON.stringify(res));
}
else {
done();
throw (new Error("Error: " + JSON.stringify(res)));
}
});

});


Expand Down
Loading