[Feature] Event ID implementation (#374)

* event id implementation

* removed unused var

* removed polyfill

* event id logic change

* omitting internal event keys from truncation

* scroll fix and tests

* Update CHANGELOG.md

* eventIDoverride

---------

Co-authored-by: ArtursKadikis <kadikis.arturs@gmail.com>

Author: turtledreams

CHANGELOG.md

@@ -1,6 +1,9 @@
 ## 23.02.0
-- Fixed a bug where previous session cookie persisted even when the 'clear_stored_id' flag was set to true
+- Events are now recorded with an internal ID.
+- Mitigated an issue where users could have truncate an internal event key
+- Mitigated an issue where SDK was reporting empty scroll values
 - Mitigated an issue that caused the device changes, after init, to not reflect on memory
+- Fixed a bug where previous session cookie persisted even when the 'clear_stored_id' flag was set to true
 
 ## 22.06.5
 - SDK now adds userAgent string to each request to prevent proxy related issues

cypress/integration/events.js

@@ -53,7 +53,7 @@ describe("Events tests ", () => {
             Countly.add_event(eventObj);
             cy.fetch_local_event_queue().then((eq) => {
                 expect(eq.length).to.equal(1);
-                cy.check_event(eq[0], eventObj);
+                cy.check_event(eq[0], eventObj, undefined, false);
             });
         });
     });
@@ -74,7 +74,7 @@ describe("Events tests ", () => {
                     cy.fetch_local_event_queue().then((eq) => {
                         expect(eq.length).to.equal(1);
                         // we waited 3000 milliseconds so duration must be 3 to 4
-                        cy.check_event(eq[0], timedEventObj, 4);
+                        cy.check_event(eq[0], timedEventObj, 4, false);
                     });
                 });
             });
@@ -97,7 +97,7 @@ describe("Events tests ", () => {
                     cy.fetch_local_event_queue().then((eq) => {
                         expect(eq.length).to.equal(1);
                         // we waited 3000 milliseconds so duration must be 3 to 4
-                        cy.check_event(eq[0], timedEventObjLong, 4);
+                        cy.check_event(eq[0], timedEventObjLong, 4, false);
                     });
                 });
             });
@@ -150,7 +150,7 @@ describe("Events tests ", () => {
                 cy.fetch_local_event_queue().then((eq) => {
                     expect(eq.length).to.equal(1);
                     // we waited 3000 milliseconds so duration must be 3 to 4
-                    cy.check_event(eq[0], timedEventObj, 3);
+                    cy.check_event(eq[0], timedEventObj, 3, false);
                 });
             });
         });
@@ -168,7 +168,7 @@ describe("Events tests ", () => {
                 cy.fetch_local_event_queue().then((eq) => {
                     expect(eq.length).to.equal(1);
                     // we waited 3000 milliseconds so duration must be 3 to 4
-                    cy.check_event(eq[0], timedEventObj, 3);
+                    cy.check_event(eq[0], timedEventObj, 3, false);
                 });
             });
         });
@@ -202,7 +202,7 @@ describe("Events tests ", () => {
                 cy.fetch_local_event_queue().then((eq) => {
                     expect(eq.length).to.equal(1);
                     // we waited 3000 milliseconds so duration must be 3 to 4
-                    cy.check_event(eq[0], timedEventObjLong, 3);
+                    cy.check_event(eq[0], timedEventObjLong, 3, false);
                 });
             });
         });

cypress/integration/heatmaps.js

@@ -26,13 +26,14 @@ describe("Browser heatmap tests, scrolls", () => {
             expect(rq.length).to.equal(4);
             // first object of the queue should be about begin session, second is orientation
             cy.check_session(rq[0], undefined, undefined, hp.appKey);
+            cy.check_view_event(JSON.parse(rq[1].events)[1], "/cypress/fixtures/scroll_test.html", undefined, false);
             // third object of the queue should be about session extension, also input the expected duration range, we expect 0 here so we enter a value lower than that but not deviated more than 1
             cy.check_session(rq[2], -0.5, undefined);
             // fourth object of the queue should be events in the queue, there must be 4 of them
-            cy.check_view_event(JSON.parse(rq[3].events)[0], "/cypress/fixtures/scroll_test.html", 0);
+            cy.check_view_event(JSON.parse(rq[3].events)[0], "/cypress/fixtures/scroll_test.html", 0, false);
             cy.check_scroll_event(JSON.parse(rq[3].events)[1]);
             // number 3 is orientation
-            cy.check_view_event(JSON.parse(rq[3].events)[3], "/cypress/fixtures/scroll_test_2.html");
+            cy.check_view_event(JSON.parse(rq[3].events)[3], "/cypress/fixtures/scroll_test_2.html", undefined, false); // new page not new view
         });
     });
     it("Check if scrolls are sent if for single page apps/sites", () => {
@@ -44,21 +45,26 @@ describe("Browser heatmap tests, scrolls", () => {
         cy.scrollTo("bottom");
         // click button that triggers view change
         cy.get("#b2").click();
-        // 2 request with 1 session and 1 events
-        cy.fetch_local_request_queue(hp.appKey).then((rq) => {
-            cy.log(rq);
-            expect(rq.length).to.equal(2);
-            cy.check_session(rq[0], undefined, undefined, hp.appKey);
-        });
-        // 6 events with 4 views and 2 scrolls must be here
-        cy.fetch_local_event_queue(hp.appKey).then((eq) => {
-            cy.log(eq);
-            cy.check_scroll_event(eq[0]);
-            cy.check_view_event(eq[1], "/cypress/fixtures/scroll_test_3.html", 0);
-            cy.check_view_event(eq[2], "v1");
-            cy.check_scroll_event(eq[3]);
-            cy.check_view_event(eq[4], "v1", 0);
-            cy.check_view_event(eq[5], "v2");
+        // There should be 3 requests: session -> event batch 1 -> event batch 2
+        hp.waitFunction(hp.getTimestampMs(), 1000, 100, () => {
+            cy.fetch_local_request_queue(hp.appKey).then((rq) => {
+                expect(rq.length).to.equal(3);
+
+                cy.check_session(rq[0], undefined, undefined, hp.appKey);
+
+                const eventBatch1 = JSON.parse(rq[1].events); // 0 is orientation, 1 is view
+                expect(eventBatch1[0].key).to.equal("[CLY]_orientation");
+                expect(eventBatch1[0].segmentation.mode).to.be.ok;
+                cy.check_view_event(eventBatch1[1], "/cypress/fixtures/scroll_test_3.html", undefined, false);
+
+                const eventBatch2 = JSON.parse(rq[2].events); // 0 is view, 1 is scroll, 2 is view, 3 is scroll
+                cy.check_scroll_event(eventBatch2[0]);
+                cy.check_view_event(eventBatch2[1], "/cypress/fixtures/scroll_test_3.html", 0, false);
+                cy.check_view_event(eventBatch2[2], "v1", undefined, true);
+                cy.check_scroll_event(eventBatch2[3]);
+                cy.check_view_event(eventBatch2[4], "v1", 0, true);
+                cy.check_view_event(eventBatch2[5], "v2", undefined, true);
+            });
         });
     });
 });
@@ -116,7 +122,7 @@ describe("Browser heatmap tests, clicks", () => {
             cy.check_commons(clickEv[0]);
 
             const seg = clickEv[0].segmentation;
-            click_check(seg, 80, 8);
+            click_check(seg, 79, 8);
         });
     });
 });

cypress/integration/internal_limits.js

@@ -99,9 +99,13 @@ describe("Internal limit tests ", () => {
         hp.haltAndClearStorage(() => {
             initMain();
             Countly.track_pageview(viewName);
-            cy.fetch_local_event_queue().then((eq) => {
+            cy.fetch_local_event_queue().then((eq) => { // TODO: when view event is truncated we provide cvid instead of pvid. fix this
+                cy.log(eq);
                 expect(eq.length).to.equal(1);
                 cy.check_view_event_limit(eq[0], viewName, limits);
+                expect(eq[0].id).to.be.ok;
+                expect(eq[0].id.length).to.equal(21);
+                expect(eq[0].pvid).to.equal("");
             });
         });
     });

cypress/integration/sessions.js

@@ -88,7 +88,7 @@ describe("Browser session tests, auto", () => {
             // third object of the queue should be about session extension, also input the expected duration
             cy.check_session(rq[2], 5, undefined, app_key);
             // fourth object of the queue should be about event sent
-            cy.check_event(JSON.parse(rq[3].events)[0], eventObj);
+            cy.check_event(JSON.parse(rq[3].events)[0], eventObj, undefined, false);
             // fifth object of the queue should be about session extension, also input the expected duration
             cy.check_session(rq[4], 1, undefined, app_key);
         });
@@ -110,7 +110,7 @@ describe("Browser session tests, manual 1", () => {
             // third object of the queue should be about session extension, also input the expected duration
             cy.check_session(rq[2], 5, undefined, app_key);
             // fourth object of the queue should be about event sent
-            cy.check_event(JSON.parse(rq[3].events)[0], eventObj);
+            cy.check_event(JSON.parse(rq[3].events)[0], eventObj, undefined, false);
             // fifth object of the queue should be about session extension, also input the expected duration
             cy.check_session(rq[4], 1, undefined, app_key);
         });
@@ -130,7 +130,7 @@ describe("Browser session tests, manual 2", () => {
             // third object of the queue should be about session extension, also input the expected duration
             cy.check_session(rq[2], 5, undefined, app_key);
             // fourth object of the queue should be about event sent
-            cy.check_event(JSON.parse(rq[3].events)[0], eventObj);
+            cy.check_event(JSON.parse(rq[3].events)[0], eventObj, undefined, false);
             // fifth object of the queue should be about session extension, also input the expected duration
             cy.check_session(rq[4], 1, undefined, app_key);
         });
@@ -151,7 +151,7 @@ describe("Browser session tests, auto, no cookie", () => {
             // third object of the queue should be about session extension, also input the expected duration
             cy.check_session(rq[2], 5, undefined, app_key);
             // fourth object of the queue should be about event sent
-            cy.check_event(JSON.parse(rq[3].events)[0], eventObj);
+            cy.check_event(JSON.parse(rq[3].events)[0], eventObj, undefined, false);
             // fifth object of the queue should be about session extension, also input the expected duration
             cy.check_session(rq[4], 1, true, app_key);
         });
@@ -173,7 +173,7 @@ describe("Browser session tests, manual 1, no cookie", () => {
             // third object of the queue should be about session extension, also input the expected duration
             cy.check_session(rq[2], 5, undefined, app_key);
             // fourth object of the queue should be about event sent
-            cy.check_event(JSON.parse(rq[3].events)[0], eventObj);
+            cy.check_event(JSON.parse(rq[3].events)[0], eventObj, undefined, false);
             // fifth object of the queue should be about session extension, also input the expected duration
             cy.check_session(rq[4], 1, true, app_key);
         });
@@ -193,7 +193,7 @@ describe("Browser session tests, manual 2, no cookie", () => {
             // third object of the queue should be about session extension, also input the expected duration
             cy.check_session(rq[2], 5, undefined, app_key);
             // fourth object of the queue should be about event sent
-            cy.check_event(JSON.parse(rq[3].events)[0], eventObj);
+            cy.check_event(JSON.parse(rq[3].events)[0], eventObj, undefined, false);
             // fifth object of the queue should be about session extension, also input the expected duration
             cy.check_session(rq[4], 1, true, app_key);
         });

cypress/integration/user_details.js

@@ -55,13 +55,13 @@ describe("User details tests ", () => {
             Countly.add_event(eventObj);
             cy.fetch_local_event_queue().then((eq) => { // event should be in event queue
                 expect(eq.length).to.equal(1);
-                cy.check_event(eq[0], eventObj);
+                cy.check_event(eq[0], eventObj, undefined, false);
             });
             cy.wait(1000).then(() => {
                 Countly.user_details(userDetailObj);
                 cy.fetch_local_request_queue().then((rq) => { // events and user details must be here
                     expect(rq.length).to.equal(2);
-                    cy.check_event(JSON.parse(rq[0].events)[0], eventObj);
+                    cy.check_event(JSON.parse(rq[0].events)[0], eventObj, undefined, false);
                     cy.check_user_details(rq[1], userDetailObj);
                 });
                 cy.fetch_local_event_queue().then((eq) => { // event queue should be empty
@@ -427,7 +427,7 @@ describe("User details tests ", () => {
             Countly.add_event(eventObj);
             cy.fetch_local_event_queue().then((eq) => { // event should be in event queue
                 expect(eq.length).to.equal(1);
-                cy.check_event(eq[0], eventObj);
+                cy.check_event(eq[0], eventObj, undefined, false);
             });
             cy.wait(1000).then(() => {
                 Countly.userData.set("key", "value");
@@ -436,7 +436,7 @@ describe("User details tests ", () => {
                 });
                 cy.fetch_local_event_queue().then((eq) => { // event should be in event queue
                     expect(eq.length).to.equal(1);
-                    cy.check_event(eq[0], eventObj);
+                    cy.check_event(eq[0], eventObj, undefined, false);
                 });
             });
         });
@@ -447,15 +447,15 @@ describe("User details tests ", () => {
             Countly.add_event(eventObj);
             cy.fetch_local_event_queue().then((eq) => { // event should be in event queue
                 expect(eq.length).to.equal(1);
-                cy.check_event(eq[0], eventObj);
+                cy.check_event(eq[0], eventObj, undefined, false);
             });
             cy.wait(1000).then(() => {
                 Countly.userData.set("key", "value");
                 Countly.userData.save();
 
                 cy.fetch_local_request_queue().then((rq) => {
                     expect(rq.length).to.equal(2);
-                    cy.check_event(JSON.parse(rq[0].events)[0], eventObj);
+                    cy.check_event(JSON.parse(rq[0].events)[0], eventObj, undefined, false);
                     const custom = JSON.parse(rq[1].user_details).custom;
                     expect(Object.keys(custom).length).to.equal(1);
                     expect(custom.key).to.equal("value");

cypress/integration/views.js

@@ -15,14 +15,29 @@ function initMain() {
 var pageNameOne = "test view page name1";
 var pageNameTwo = "test view page name2";
 
-describe("Views tests ", () => {
+describe("View ID tests ", () => {
+    it("Checks if UUID and secureRandom works as intended", () => {
+        hp.haltAndClearStorage(() => {
+            initMain();
+            const uuid = Countly._internals.generateUUID();
+            const id = Countly._internals.secureRandom();
+            assert.equal(uuid.length, 36);
+            assert.equal(id.length, 21);
+            const uuid2 = Countly._internals.generateUUID();
+            const id2 = Countly._internals.secureRandom();
+            assert.equal(uuid2.length, 36);
+            assert.equal(id2.length, 21);
+            assert.notEqual(uuid, uuid2);
+            assert.notEqual(id, id2);
+        });
+    });
     it("Checks if recording page view works", () => {
         hp.haltAndClearStorage(() => {
             initMain();
             Countly.track_view(pageNameOne);
             cy.fetch_local_event_queue().then((eq) => {
                 expect(eq.length).to.equal(1);
-                cy.check_view_event(eq[0], pageNameOne);
+                cy.check_view_event(eq[0], pageNameOne, undefined, false);
             });
         });
     });
@@ -33,10 +48,20 @@ describe("Views tests ", () => {
             cy.wait(3000).then(() => {
                 Countly.track_view(pageNameOne);
                 cy.fetch_local_event_queue().then((eq) => {
+                    cy.log(eq);
                     expect(eq.length).to.equal(3);
-                    cy.check_view_event(eq[0], pageNameOne);
-                    cy.check_view_event(eq[1], pageNameOne, 3);
-                    cy.check_view_event(eq[2], pageNameOne);
+                    cy.check_view_event(eq[0], pageNameOne, undefined, false);
+                    const id1 = eq[0].id;
+
+                    cy.check_view_event(eq[1], pageNameOne, 3, false);
+                    const id2 = eq[1].id;
+                    assert.equal(id1, id2);
+
+                    cy.check_view_event(eq[2], pageNameOne, undefined, true);
+                    const id3 = eq[2].id;
+                    const pvid = eq[2].pvid;
+                    assert.equal(id1, pvid);
+                    assert.notEqual(id3, pvid);
                 });
             });
         });
@@ -45,18 +70,66 @@ describe("Views tests ", () => {
         hp.haltAndClearStorage(() => {
             initMain();
             Countly.track_view(pageNameOne);
-            var expectedDur = 4000;
-            hp.waitFunction(hp.getTimestampMs(), expectedDur, 500, ()=>{
-                // cy.wait(4000).then(() => {
+            hp.waitFunction(hp.getTimestampMs(), 4000, 500, ()=>{
                 Countly.track_view(pageNameTwo);
                 cy.fetch_local_event_queue().then((eq) => {
                     expect(eq.length).to.equal(3);
-                    cy.check_view_event(eq[0], pageNameOne);
+                    cy.check_view_event(eq[0], pageNameOne, undefined, false);
+                    const id1 = eq[0].id;
+
                     // this test is flaky we are expecting 3 and +1 (4) to make test more reliable 
-                    cy.check_view_event(eq[1], pageNameOne, expectedDur / 1000);
-                    cy.check_view_event(eq[2], pageNameTwo);
+                    cy.check_view_event(eq[1], pageNameOne, 4, false);
+                    const id2 = eq[1].id;
+                    assert.equal(id1, id2);
+
+                    cy.check_view_event(eq[2], pageNameTwo, undefined, true);
+                    const id3 = eq[2].id;
+                    const pvid = eq[2].pvid;
+                    assert.equal(id1, pvid);
+                    assert.notEqual(id3, pvid);
                 });
-                // });
+            });
+        });
+    });
+
+    // ===========================
+    //  Confirms:
+    // view A's id and event A's cvid are same
+    // view B's id and event B's cvid are same. Also view B's pvid and view A's id are same
+    // view C's id and event C's cvid are same. Also view C's pvid and view B's id are same
+    //
+    // request order: view A start -> event A -> view A end -> view B start -> event B -> view B end -> view C start -> event C  
+    // ===========================
+    it("Checks a sequence of events and page views", () => {
+        hp.haltAndClearStorage(() => {
+            initMain();
+            Countly.track_view("A");
+            Countly.add_event({ key: "A" });
+            Countly.track_view("B");
+            Countly.add_event({ key: "B" });
+            Countly.track_view("C");
+            Countly.add_event({ key: "C" });
+
+            cy.fetch_local_event_queue().then((eq) => {
+                expect(eq.length).to.equal(8);
+                cy.log(eq);
+
+                // event A and view A
+                cy.check_view_event(eq[0], "A", undefined, false); // no pvid
+                const idA = eq[0].id; // idA
+                cy.check_event(eq[1], { key: "A" }, undefined, idA); // cvid should be idA
+                cy.check_view_event(eq[2], "A", 0, false); // no pvid
+
+                // event B and view B
+                cy.check_view_event(eq[3], "B", undefined, idA); // pvid is idA
+                const idB = eq[3].id; // idB
+                cy.check_event(eq[4], { key: "B" }, undefined, idB); // cvid should be idB
+                cy.check_view_event(eq[5], "B", 0, idA); // pvid is idA
+
+                // event C and view C
+                cy.check_view_event(eq[6], "C", undefined, idB); // pvid is idB
+                const idC = eq[6].id; // idC
+                cy.check_event(eq[7], { key: "C" }, undefined, idC); // cvid should be idC  
             });
         });
     });