Added Statistics Page (RocketMap#2159)

* Added a statistics (/stats) page to show the details of what, and how often, Pokemon have been seen by the map. The page auto-refreshes to keep up to date information and can be sorted by number of times the Pokemon has been seen, Pokemon ID and Pokemon Name. Can view details for varying durations ranging from the last hour to the entire lifetime of the map.

* More information on the seen Pokemon

* Added Full Stats Link

* Potential fix for Lifetime Na times

* Potential fix for missing Pokemon

* Bug fixes & General performance improvements

* Updated Additional Info

* Couple more bug fixes

* Heat map & Fixes

* Load Overlay on page load if # specified

* New look

Gruntfile.js

@@ -10,7 +10,8 @@ module.exports = function(grunt) {
             'static/sass/main.scss',
             'static/sass/pokemon-sprite.scss'
           ],
-          'static/dist/css/mobile.built.css': 'static/sass/mobile.scss'
+          'static/dist/css/mobile.built.css': 'static/sass/mobile.scss',
+          'static/dist/css/statistics.built.css': 'static/css/statistics.css'
         }
       }
     },
@@ -36,7 +37,8 @@ module.exports = function(grunt) {
           'static/dist/js/app.built.js': 'static/js/app.js',
           'static/dist/js/map.built.js': 'static/js/map.js',
           'static/dist/js/mobile.built.js': 'static/js/mobile.js',
-          'static/dist/js/stats.built.js': 'static/js/stats.js'
+          'static/dist/js/stats.built.js': 'static/js/stats.js',
+          'static/dist/js/statistics.built.js': 'static/js/statistics.js'
         }
       }
     },
@@ -53,7 +55,8 @@ module.exports = function(grunt) {
           'static/dist/js/app.min.js': 'static/dist/js/app.built.js',
           'static/dist/js/map.min.js': 'static/dist/js/map.built.js',
           'static/dist/js/mobile.min.js': 'static/dist/js/mobile.built.js',
-          'static/dist/js/stats.min.js': 'static/dist/js/stats.built.js'
+          'static/dist/js/stats.min.js': 'static/dist/js/stats.built.js',
+          'static/dist/js/statistics.min.js': 'static/dist/js/statistics.built.js'
         }
       }
     },
@@ -103,7 +106,8 @@ module.exports = function(grunt) {
       build: {
         files: {
           'static/dist/css/app.min.css': 'static/dist/css/app.built.css',
-          'static/dist/css/mobile.min.css': 'static/dist/css/mobile.built.css'
+          'static/dist/css/mobile.min.css': 'static/dist/css/mobile.built.css',
+          'static/dist/css/statistics.min.css': 'static/dist/css/statistics.built.css'
         }
       }
     }

pogom/app.py

@@ -10,6 +10,8 @@
 from datetime import datetime
 from s2sphere import *
 from pogom.utils import get_args
+from datetime import timedelta
+from collections import OrderedDict
 
 from . import config
 from .models import Pokemon, Gym, Pokestop, ScannedLocation
@@ -30,6 +32,7 @@ def __init__(self, import_name, **kwargs):
         self.route("/mobile", methods=['GET'])(self.list_pokemon)
         self.route("/search_control", methods=['GET'])(self.get_search_control)
         self.route("/search_control", methods=['POST'])(self.post_search_control)
+        self.route("/stats", methods=['GET'])(self.get_stats)
 
     def set_search_control(self, control):
         self.search_control = control
@@ -96,6 +99,15 @@ def raw_data(self):
             d['scanned'] = ScannedLocation.get_recent(swLat, swLng, neLat,
                                                       neLng)
 
+        if request.args.get('seen', 'false') == 'true':
+            for duration in self.get_valid_stat_input()["duration"]["items"].values():
+                if duration["selected"] == "SELECTED":
+                    d['seen'] = Pokemon.get_seen(duration["value"])
+                    break
+
+        if request.args.get('appearances', 'false') == 'true':
+            d['appearances'] = Pokemon.get_appearances(request.args.get('pokemonid'), request.args.get('last', type=float))
+
         return jsonify(d)
 
     def loc(self):
@@ -166,6 +178,50 @@ def list_pokemon(self):
                                origin_lat=lat,
                                origin_lng=lon)
 
+    def get_valid_stat_input(self):
+        duration = request.args.get("duration", type=str)
+        sort = request.args.get("sort", type=str)
+        order = request.args.get("order", type=str)
+        valid_durations = OrderedDict()
+        valid_durations["1h"] = {"display": "Last Hour", "value": timedelta(hours=1), "selected": ("SELECTED" if duration == "1h" else "")}
+        valid_durations["3h"] = {"display": "Last 3 Hours", "value": timedelta(hours=3), "selected": ("SELECTED" if duration == "3h" else "")}
+        valid_durations["6h"] = {"display": "Last 6 Hours", "value": timedelta(hours=6), "selected": ("SELECTED" if duration == "6h" else "")}
+        valid_durations["12h"] = {"display": "Last 12 Hours", "value": timedelta(hours=12), "selected": ("SELECTED" if duration == "12h" else "")}
+        valid_durations["1d"] = {"display": "Last Day", "value": timedelta(days=1), "selected": ("SELECTED" if duration == "1d" else "")}
+        valid_durations["7d"] = {"display": "Last 7 Days", "value": timedelta(days=7), "selected": ("SELECTED" if duration == "7d" else "")}
+        valid_durations["14d"] = {"display": "Last 14 Days", "value": timedelta(days=14), "selected": ("SELECTED" if duration == "14d" else "")}
+        valid_durations["1m"] = {"display": "Last Month", "value": timedelta(days=365/12), "selected": ("SELECTED" if duration == "1m" else "")}
+        valid_durations["3m"] = {"display": "Last 3 Months", "value": timedelta(days=3*365/12), "selected": ("SELECTED" if duration == "3m" else "")}
+        valid_durations["6m"] = {"display": "Last 6 Months", "value": timedelta(days=6*365/12), "selected": ("SELECTED" if duration == "6m" else "")}
+        valid_durations["1y"] = {"display": "Last Year", "value": timedelta(days=365), "selected": ("SELECTED" if duration == "1y" else "")}
+        valid_durations["all"] = {"display": "Map Lifetime", "value": 0, "selected": ("SELECTED" if duration == "all" else "")}
+        if duration not in valid_durations:
+            valid_durations["1d"]["selected"] = "SELECTED"
+        valid_sort = OrderedDict()
+        valid_sort["count"] = {"display": "Count", "selected": ("SELECTED" if sort == "count" else "")}
+        valid_sort["id"] = {"display": "Pokedex Number", "selected": ("SELECTED" if sort == "id" else "")}
+        valid_sort["name"] = {"display": "Pokemon Name", "selected": ("SELECTED" if sort == "name" else "")}
+        if sort not in valid_sort:
+            valid_sort["count"]["selected"] = "SELECTED"
+        valid_order = OrderedDict()
+        valid_order["asc"] = {"display": "Ascending", "selected": ("SELECTED" if order == "asc" else "")}
+        valid_order["desc"] = {"display": "Descending", "selected": ("SELECTED" if order == "desc" else "")}
+        if order not in valid_order:
+            valid_order["desc"]["selected"] = "SELECTED"
+        valid_input = OrderedDict()
+        valid_input["duration"] = {"display": "Duration", "items": valid_durations}
+        valid_input["sort"] = {"display": "Sort", "items": valid_sort}
+        valid_input["order"] = {"display": "Order", "items": valid_order}
+        return valid_input
+
+    def get_stats(self):
+        return render_template('statistics.html',
+                               lat=self.current_location[0],
+                               lng=self.current_location[1],
+                               gmaps_key=config['GMAPS_KEY'],
+                               valid_input=self.get_valid_stat_input()
+                               )
+
 
 class CustomJSONEncoder(JSONEncoder):
 

pogom/models.py

@@ -6,7 +6,7 @@
 import calendar
 from peewee import Model, SqliteDatabase, InsertQuery,\
                    IntegerField, CharField, DoubleField, BooleanField,\
-                   DateTimeField, OperationalError, create_model_tables
+                   DateTimeField, OperationalError, create_model_tables, fn
 from playhouse.flask_utils import FlaskDB
 from playhouse.pool import PooledMySQLDatabase
 from playhouse.shortcuts import RetryOperationalError
@@ -135,6 +135,52 @@ def get_active_by_id(cls, ids, swLat, swLng, neLat, neLng):
 
         return pokemons
 
+    @classmethod
+    def get_seen(cls, timediff):
+        if timediff:
+            timediff = datetime.utcnow() - timediff
+        pokemon_count_query = (Pokemon
+                               .select(Pokemon.pokemon_id,
+                                       fn.COUNT(Pokemon.pokemon_id).alias('count'),
+                                       fn.MAX(Pokemon.disappear_time).alias('lastappeared')
+                                       )
+                               .where(Pokemon.disappear_time > timediff)
+                               .group_by(Pokemon.pokemon_id)
+                               .alias('counttable')
+                               )
+        query = (Pokemon
+                 .select(Pokemon.pokemon_id,
+                         Pokemon.disappear_time,
+                         Pokemon.latitude,
+                         Pokemon.longitude,
+                         pokemon_count_query.c.count)
+                 .join(pokemon_count_query, on=(Pokemon.pokemon_id == pokemon_count_query.c.pokemon_id))
+                 .where(Pokemon.disappear_time == pokemon_count_query.c.lastappeared)
+                 .dicts()
+                 )
+        pokemons = []
+        total = 0
+        for p in query:
+            p['pokemon_name'] = get_pokemon_name(p['pokemon_id'])
+            pokemons.append(p)
+            total += p['count']
+
+        return {'pokemon': pokemons, 'total': total}
+
+    @classmethod
+    def get_appearances(cls, pokemon_id, last_appearance):
+        query = (Pokemon
+                 .select()
+                 .where((Pokemon.pokemon_id == pokemon_id) &
+                        (Pokemon.disappear_time > datetime.utcfromtimestamp(last_appearance/1000.0))
+                        )
+                 .order_by(Pokemon.disappear_time.asc())
+                 .dicts()
+                 )
+        appearances = []
+        for a in query:
+            appearances.append(a)
+        return appearances
 
 class Pokestop(BaseModel):
     pokestop_id = CharField(primary_key=True, max_length=50)

static/css/statistics.css

@@ -0,0 +1,156 @@
+@charset "UTF-8";
+
+body{
+    background: #f1f1f1;
+    z-index: 0;
+    overflow: auto;
+}
+
+table{
+    background: #f8f8f8;
+}
+
+table thead{
+    font-weight: bold;
+}
+
+table thead .title{
+    text-align: center;
+}
+
+table .even{
+    background: #f1f1f1;
+}
+
+table tfoot{
+    font-weight: bold;
+}
+
+table td{
+    padding: 5px;
+}
+
+.label{
+    font-weight: bold;
+}
+
+.totals{
+    margin-top: 3.5em;
+    margin-left: 10px;
+    margin-right: 10px;
+    text-transform: uppercase;
+}
+
+.options{
+    padding-bottom: 5px;
+}
+
+.container{
+    overflow: auto;
+    line-height: 20px;
+    font-size: 15px;
+    margin-left: 10px;
+    margin-right: 10px;
+}
+
+.container .item{
+    float: left;
+    border: 1px solid #acacac;
+    border-radius: 5px;
+    -moz-border-radius: 5px;
+    background: #f8f8f8;
+    margin: 0px 5px 5px 0px;
+    padding: 5px;
+    overflow: auto;
+    height: 130px;
+    min-width: 260px;
+}
+
+.container .item .image{
+    float: left;
+    width: 40px;
+    text-align: center;
+}
+
+.container .item .name{
+    height: 30px;
+    line-height: 30px;
+}
+
+.container .item .name a{
+    font-weight: bold;
+}
+
+.overlay{
+    z-index: 100000;
+    display: none;
+    background-color: rgba(0,0,0,0.5);
+    position: fixed;
+    top: 0;
+    left: 0;
+    width: 100%;
+    min-height: 100%;
+}
+
+#location_map{
+    height: 100%;
+}
+
+#times_list{
+    float: left;
+    background: white;
+    position: fixed;
+    top: 60px;
+    bottom: 60px;
+    width: 200px;
+    z-index: 9999;
+    overflow: auto;
+}
+
+#times_list div{
+    padding-bottom: 2px;
+}
+
+#times_list .row0{
+    background: #f1f1f1;
+}
+
+#times_list .row1{
+    background: #f8f8f8;
+}
+
+.overlay .close{
+    position: relative;
+    float: right;
+    margin-top: 20px;
+    margin-right: 100px;
+    background: white;
+    cursor: pointer;
+}
+
+.overlay .content{
+    background: white;
+    margin: 50px;
+    border-radius: 10px;
+    -moz-border-radius: 10px;
+    padding: 10px;
+    position: fixed;
+    top: 0px;
+    bottom: 0px;
+    left: 0px;
+    right: 0px;
+}
+
+#loading{
+    width: 100%;
+    height: 100%;
+    position: fixed;
+    top: 0px;
+}
+
+#loading img{
+    top: 50%;
+    left: 50%;
+    margin-right: -50%;
+    position: absolute;
+}