Merge pull request #13 from Countly/next

Next

.eslintignore

@@ -5,11 +5,11 @@ frontend/express/public/javascripts/min/**
 frontend/express/public/javascripts/utils/**
 frontend/express/public/javascripts/visualization/**
 frontend/express/public/themes/**
+frontend/express/public/sdk/**
 plugins/*/frontend/public/*
 plugins/*/frontend/public/javascripts/*
 bin/scripts/nghttp2/*
 plugins/push/api/parts/apn/*
 
 !plugins/*/frontend/public/javascripts
-!plugins/*/frontend/public/javascripts/countly.models.js
-!plugins/*/frontend/public/javascripts/countly.views.js
+!plugins/*/frontend/public/javascripts/countly.*.js

.eslintrc.json

@@ -1,6 +1,7 @@
 {
     "extends": "eslint:recommended",
     "rules": {
+        "require-atomic-updates": "off",
         "block-spacing": [
             "error",
             "always"
@@ -28,10 +29,7 @@
             "error",
             "all"
         ],
-        "eol-last": [
-            "error",
-            "never"
-        ],
+        "eol-last": "off",
         "func-call-spacing": [
             "error",
             "never"
@@ -126,8 +124,7 @@
         {
             "files": [
                 "frontend/express/public/javascripts/countly/*.js",
-                "plugins/*/frontend/public/javascripts/countly.models.js",
-                "plugins/*/frontend/public/javascripts/countly.views.js" 
+                "plugins/*/frontend/public/javascripts/countly.*.js"
             ],
             "env": {
                 "browser": true
@@ -197,14 +194,15 @@
                 "frontend/express/libs/*.js",
                 "plugins/pluginManager.js",
                 "plugins/*/api/**/*.js",
-                "plugins/*/frontend/*.js"
+                "plugins/*/frontend/*.js",
+                "plugins/*/extend/*.js"
             ],
             "env": {
                 "es6": true,
                 "node": true
             },
             "parserOptions": {
-                "ecmaVersion": 2017
+                "ecmaVersion": 9
             },
             "rules": {
                 "no-console": "off",
@@ -281,7 +279,7 @@
                 "node": true
             },
             "parserOptions": {
-                "ecmaVersion": 2017
+                "ecmaVersion": 9
             },
             "rules": {
                 "no-console": "off"
@@ -299,7 +297,7 @@
                 "mocha": true
             },
             "parserOptions": {
-                "ecmaVersion": 2017
+                "ecmaVersion": 9
             },
             "rules": {
                 "no-console": "off",

.travis.yml

@@ -1,8 +1,9 @@
 language: nodejs
-node_js: "8"
+dist: trusty
+node_js: "10"
 env:
   global:
-  - MONGODB=3.6.6
+  - MONGODB=4.0.12
   - secure: ZXdUMP+8Nk9Kd5nX31D/DIB1t2xlJd+T4W7umWjwpy+KPA9d1W1OhUG6V2Gpv3PsJf+UtMosgZsywcLFVG4l/WcjAhs3c9pegW5ZRYAEfVjkk7aHhWQGvj5DID8iAAxs91Mm8LwJ43o68x+XN6746zArPqcVzQltDunJ0G3gfh8=
 branches:
   except:

CHANGELOG.md

@@ -1,3 +1,132 @@
+## Version 19.08
+
+**Fixes**
+* [api] fixed error on deleting user with exported data
+* [api] improved tops speed with aggregation pipeline
+* [app_versions] fixing displaying empty state
+* [assistant] handling case, when document could not be read
+* [browser] correctly convert metric to collection for correct data
+* [configs] display correct values in app configuration after save
+* [events] fixed event overview incorrectly formats duration
+* [frontend] added configuration element for google maps api key
+* [frontend] correct event total calculation for segmented view of aggregated data
+* [frontend] fixed Drop throws error when datatable is empty
+* [frontend] fixed japanese locale dates
+* [frontend] fixed refreshing total user correction for today
+* [frontend] reduced session extend calls.
+* [loyalty] fix table ordering
+* [push] Correct system log on push credentials update error
+* [push] Fixing populator creating messages for wrong app
+* [slipping-away] fix fetch user list bug
+* [source] fixed localization key in configs
+* [star-rating] lots of bug and ui fixes
+* [ui] Fixing expand row icon switches on refresh
+* [ui] changed email reports % colors.
+* [ui] disabling annoying chrome autocomplete
+* [ui] fix bug with Ubuntu font on Windows
+* [ui] prevent double scrolling
+* [ui] set app title attribute when switching apps
+* [ui] users and new users country sorting problem resolved
+* [views] fixes with tokens expiring when viewing heatmaps
+
+**Enterprise fixes**
+* [attribution] encode link if it is not a custom scheme
+* [attribution] fixed using last campaign urls when creating new campaign
+* [block] fixed creating empty users for new blocked users
+* [cohorts] fixed editing cohorts with big lists
+* [cohorts] handle empy user sets correctly and more efficiently
+* [dashboards] block all popups for dashboard image rendering in emails
+* [dashboards] fixed sending email reports if creator is deleted
+* [dashboards] improve screenshot rendering speed with cache for emails
+* [drill] actions button avaialble without query
+* [drill] better bucket and value selection for report manager data
+* [drill] big list values are not selected when replaying query bugfix
+* [drill] cohorts querying fixes
+* [drill] correct user count for multi BY queries
+* [drill] fixed generating month ticks for more than 1 year
+* [drill] query performance improved with new indexes
+* [flows] fix flows logic bug about app data clearing
+* [flows] fix views event logic bug
+* [funnels] allow to create funnels without events but with views
+* [funnels] fixed editing funnels with grouping elements
+* [funnels] top percentage update bug fixed
+* [groups] add uppercase to allowed characters for name and group id
+* [revenue] fix updating overview widget on period changes
+* [whitelabeling] replacement of word Countly with company name in all localized strings
+
+**New Features**
+* [api] allow updating multiple app users
+* [applications] application lock mechanism
+* [authorization] tokens now support url parameters as limits
+* [crashes] multi thread error support
+* [crashes] new crash stack processing for android and javascript
+* [crashes] new dropdown menu UI in crash groups
+* [dbviewer] added custom field support to dbviewer sort
+* [events] display top events for event overview
+* [frontend] option to delete your own account
+* [plugins] asynchronously check plugin enabling/disabling procedure
+* [push] Adding push events to user’s timeline
+* [push] allow building target audience just before scheduled date
+* [push] option to view recipients of a push
+* [push] proxy authentication support
+* [push] real-time event triggered push notifications
+* [report-manager] smarter more often regenerating automatic reports
+* [security] added password secret salt support in configuration file
+* [slipping-away-users] add flexible periods setting support
+* [two-factor-auth] enable two factor authentication through Microsoft or Google authenticator apps
+* [ui] added native tab behaivor to cly-select element
+* [ui] allow uploading dashboard profile pictures
+* [ui] customize graph colors through config file
+* [ui] moved some management tools to top menu
+* [ui] new graph note system
+* [ui] new menu design
+* [ui] new prelogin page design
+* [views] select columns to display functionality
+
+**New Enterprise Features**
+* [ab-testing] new plugin to perform AB testing
+* [attribution] allow providing custom domain for campaign links
+* [attribution] allow reattribution
+* [concurrent_users] new and more performant and detailed version of live plugin
+* [crash_symbolication] support for native crashes and symbolication
+* [crashes-jira] plugin to tie caught crashes with JIRA issues
+* [drill] make Drill configuration on app level
+* [formulas] new plugin to perform arithmetical computations on selected metrics
+* [funnels] allow sorting steps by drag and drop when editing funnels
+* [geo] Allowing float as geo radius
+* [users] added region support
+* [users] select columns to display functionality
+
+**Development related**
+* [api] moved account deletion to backend api.
+* [api] provide cancel request on app not exists
+* [api] refactored getPeriodObj
+* [cmd] added new upgrade subcommands for automatic multiple version upgrades
+* [cmd] script to upgrade countly to ee
+* [cmd] use password prompt to mask sensitive data
+* [common] add email for validation
+* [configs] improved config parsing and usage
+* [configs] more session and cookie settings in config file
+* [core] Increasing default heap size to 2Gb for API
+* [crashes] make crash identification model independent
+* [db] log incorrect Object ID to info level
+* [example] AWS SES - Simple Email Service example
+* [example] nginx example config of blocking access outside intranet
+* [frontend] redirect rather than render on POST processing
+* [frontend] replaced old google library loader with new version
+* [frontend] set cookie only when theme is customizable on user level
+* [frontend] trust estimation correction more than new users data
+* [members] moved all member functions to single separate members utility
+* [scripts] allow countly user to enable plugins
+* [scripts] correctly backup nginx config on ubuntu
+* [scripts] update_translation.js perfs and reliability boost
+* [security] HTML escaping in all localized strings
+* [security] remove flash cross domain policy
+* [taskmanager] no need to decode html (breaks json parsing)
+* [tests] additional way to parse CSRF from body in case of minified html
+* [ui] new menu management system
+* [ui] updated font awesome lib
+
 ## Version 19.02.1
 
 **Fixes**

Gruntfile.js

@@ -55,7 +55,15 @@ module.exports = function(grunt) {
                     'frontend/express/public/javascripts/utils/vue.min.js',
                     'frontend/express/public/javascripts/utils/jquery.xss.js',
                     'frontend/express/public/javascripts/countly/countly.common.js',
-                    'frontend/express/public/javascripts/utils/simpleUpload.min.js'
+                    'frontend/express/public/javascripts/utils/simpleUpload.min.js',
+                    'frontend/express/public/javascripts/utils/jsoneditor/codemirror.js',
+                    'frontend/express/public/javascripts/utils/jsoneditor/javascript.js',
+                    'frontend/express/public/javascripts/utils/jsoneditor/json2.js',
+                    'frontend/express/public/javascripts/utils/jsoneditor/jsonlint.js',
+                    'frontend/express/public/javascripts/utils/jsoneditor/minify.json.js',
+                    'frontend/express/public/javascripts/utils/jsoneditor/jsoneditor.js',
+                    'frontend/express/public/javascripts/utils/Sortable.min.js',
+                    'frontend/express/public/javascripts/utils/vuedraggable.umd.min.js'
                 ],
                 dest: 'frontend/express/public/javascripts/min/countly.utils.concat.js'
             },
@@ -127,6 +135,7 @@ module.exports = function(grunt) {
                         'frontend/express/public/stylesheets/main.css',
                         'frontend/express/public/stylesheets/amaranjs/amaran.min.css',
                         'frontend/express/public/stylesheets/selectize/selectize.css',
+                        'frontend/express/public/stylesheets/jsoneditor/codemirror.css',
                         'frontend/express/public/javascripts/dom/tipsy/tipsy.css',
                         'frontend/express/public/javascripts/visualization/rickshaw/rickshaw.min.css',
                         'frontend/express/public/javascripts/dom/pace/pace-theme-flash.css',

README.md

@@ -3,7 +3,7 @@
 
 <p align="right">
 
-[![Build Status](https://api.travis-ci.org/Countly/countly-server.png?branch=master)](https://travis-ci.org/Countly/countly-server) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/98c2726f2d734697a5f1ac0d453f0a06)](https://app.codacy.com/app/ar2rsawseen/countly-server?utm_source=github.com&utm_medium=referral&utm_content=Countly/countly-server&utm_campaign=Badge_Grade_Dashboard) [![Install Countly on DigitalOcean](https://do.count.ly/button.svg?v2)](http://do.count.ly)
+[![Build Status](https://api.travis-ci.org/Countly/countly-server.png?branch=master)](https://travis-ci.org/Countly/countly-server) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/98c2726f2d734697a5f1ac0d453f0a06)](https://app.codacy.com/app/ar2rsawseen/countly-server?utm_source=github.com&utm_medium=referral&utm_content=Countly/countly-server&utm_campaign=Badge_Grade_Dashboard) [![Install Countly on DigitalOcean](https://count.ly/github/install-on-digital-ocean.svg)](http://do.count.ly)
 
 </p>
 
@@ -118,7 +118,7 @@ There are several ways to install Countly:
 
 1. The following command will download and install Countly on your **Ubuntu** or **CentOS** server. 
 
-    `wget -qO- http://c.ly/install | bash`
+    `wget -qO- https://c.ly/install | bash`
 
 2. If you have a Digital Ocean account, [click here and install on Digital Ocean](http://do.count.ly) with a single click.
 

api/api.js

@@ -11,6 +11,7 @@ const common = require('./utils/common.js');
 const {processRequest} = require('./utils/requestProcessor');
 const versionInfo = require('../frontend/express/version.info');
 const frontendConfig = require('../frontend/express/config.js');
+const {CacheMaster, CacheWorker} = require('./parts/data/cache.js');
 
 var t = ["countly:", "api"];
 
@@ -207,6 +208,12 @@ const passToMaster = (worker) => {
 };
 
 if (cluster.isMaster) {
+    common.cache = new CacheMaster(common.db);
+    common.cache.start().then(plugins.dispatch.bind(plugins, '/cache/init', {}), e => {
+        console.log(e);
+        process.exit(1);
+    });
+
     const workerCount = (countlyConfig.api.workers)
         ? countlyConfig.api.workers
         : os.cpus().length;
@@ -231,17 +238,22 @@ if (cluster.isMaster) {
 
     // Allow configs to load & scanner to find all jobs classes
     setTimeout(() => {
+        jobs.job('api:topEvents').replace().schedule('every 1 day');
         jobs.job('api:ping').replace().schedule('every 1 day');
         jobs.job('api:clear').replace().schedule('every 1 day');
         jobs.job('api:clearTokens').replace().schedule('every 1 day');
-        jobs.job('api:task').replace().schedule('every 59 mins starting on the 59 min');
+        jobs.job('api:task').replace().schedule('every 1 hour on the first min');
         jobs.job('api:userMerge').replace().schedule('every 1 hour on the 10th min');
     }, 10000);
 }
 else {
     console.log("Starting worker", process.pid, "parent:", process.ppid);
     const taskManager = require('./utils/taskmanager.js');
     common.db = plugins.dbConnection(countlyConfig);
+
+    common.cache = new CacheWorker(common.db);
+    common.cache.start();
+
     //since process restarted mark running tasks as errored
     taskManager.errorResults({db: common.db});
 
@@ -291,7 +303,7 @@ else {
             const headers = {};
             headers["Access-Control-Allow-Origin"] = "*";
             headers["Access-Control-Allow-Methods"] = "POST, GET, OPTIONS";
-            headers["Access-Control-Allow-Headers"] = "countly-token";
+            headers["Access-Control-Allow-Headers"] = "countly-token, Content-Type";
             res.writeHead(200, headers);
             res.end();
         }

api/jobs/ping.js

@@ -59,6 +59,7 @@ class PingJob extends job.Job {
                                 timestamp: Math.floor(date.getTime() / 1000),
                                 hour: date.getHours(),
                                 dow: date.getDay(),
+                                no_meta: true,
                                 events: JSON.stringify([
                                     {
                                         key: "PING",

api/jobs/task.js

@@ -5,7 +5,6 @@ const job = require('../parts/jobs/job.js'),
     Promise = require("bluebird");
 const common = require('../utils/common.js');
 const taskmanager = require('../utils/taskmanager.js');
-const moment = require('moment');
 
 /**
  *  Task Monitor Job extend from Countly Job
@@ -18,14 +17,44 @@ class MonitorJob extends job.Job {
      */
     run(db, done) {
         const self = this;
-        const targetHour = (new moment()).hours();
+        /**
+         * Filter tasks to run base on job run time
+         * @param {object} task object from db
+         * @return {boolean} return true if can run now
+         */
+        function tasksFilter(task) {
+            if (task.status === 'running' || task.status === 'rerunning') {
+                return false;
+            }
+            const lastStart = task.start || 0;
+            const lastEnd = task.end || 0;
+            const now = Date.now();
+            const duration = lastEnd - lastStart;
+            if (duration <= 60 * 1000 && duration >= 0) {
+                return true;
+            }
+            if (duration <= 10 * 60 * 1000 && now - lastEnd >= 2 * 60 * 60 * 1000) {
+                return true;
+            }
+            if (duration <= 30 * 60 * 1000 && now - lastEnd >= 4 * 60 * 60 * 1000) {
+                return true;
+            }
+            if (duration <= 60 * 60 * 1000 && now - lastEnd >= 12 * 60 * 60 * 1000) {
+                return true;
+            }
+            if (duration > 60 * 60 * 1000 && now - lastEnd >= 24 * 60 * 60 * 1000) {
+                return true;
+            }
+            return false;
+        }
+
         common.db.collection("long_tasks").find({
             autoRefresh: true,
-            r_hour: targetHour
         }).toArray(function(err, tasks) {
             log.d('Running Task Monitor Job ....');
             log.d("job info:", self._json, tasks);
-            tasks.forEach((task)=>{
+            const filteredTasks = tasks.filter(tasksFilter);
+            filteredTasks.forEach((task) => {
                 return Promise.coroutine(function *() { // eslint-disable-line require-yield
                     try {
                         taskmanager.rerunTask({