Added Pagination support to Bots Page (google#1982)

1. Add pagination support to bots page.
1. Requests for search and page change will be using AJAX. Added a JsonHandler in page handler.
2. Search shifted from on-browser to server side using keywords present in name. (Due to this, the existing builds' heartbeats data should be modified as a new attribute is added to Heartbeat model, provided a script for this in the migrations)
3. Added new test for changed APIs.

Screenshot:
![bots to send](https://user-images.githubusercontent.com/30122295/90531746-13187200-e194-11ea-9cab-80abb90038a9.png)

Author: iamarshsingh

src/appengine/handlers/bots.py

@@ -15,13 +15,24 @@
 
 import datetime
 
+from flask import request
+
 from base import tasks
 from base import utils
 from datastore import data_types
 from datastore import ndb_utils
 from handlers import base_handler_flask
+from libs import filters
 from libs import handler_flask
 from libs import helpers
+from libs.query import datastore_query
+
+PAGE_SIZE = 10
+MORE_LIMIT = 50 - PAGE_SIZE  # exactly 5 pages
+
+FILTERS = [
+    filters.Keyword([], 'keywords', 'q'),
+]
 
 
 def _get_alive_cutoff():
@@ -61,31 +72,29 @@ def _convert_heartbeats_to_dicts(heartbeats):
   return result
 
 
-def _get_host_workers_heartbeats():
-  """Return host worker heartbeats."""
-  page_size = 30
-
-  query = data_types.HostWorkerAssignment.query()
-  cursor = None
+def get_results():
+  """Get results for the bots page."""
+  # Return bots sorted alphabetically by bot_name
+  query = datastore_query.Query(data_types.Heartbeat)
+  query.order('bot_name', is_desc=False)
+  params = dict(request.iterparams())
+  filters.add(query, params, FILTERS)
 
-  while True:
-    results, cursor, more = query.fetch_page(page_size, start_cursor=cursor)
-    key_ids = [assignment.key.id() for assignment in results]
-    if not key_ids:
-      break
+  page = helpers.cast(request.get('page', 1), int, "'page' is not an int.")
+  items, total_pages, total_items, has_more = query.fetch_page(
+      page=page, page_size=PAGE_SIZE, projection=None, more_limit=MORE_LIMIT)
+  items = _convert_heartbeats_to_dicts(items)
+  helpers.log('Bots', helpers.VIEW_OPERATION)
 
-    worker_mapping = dict(
-        (assignment.key.id(), assignment.worker_name) for assignment in results)
-
-    for heartbeat in data_types.Heartbeat.query(
-        data_types.Heartbeat.bot_name.IN(key_ids)):
-      heartbeat.bot_name = '{host_name} ({worker_name})'.format(
-          host_name=heartbeat.bot_name,
-          worker_name=worker_mapping[heartbeat.bot_name])
-      yield heartbeat
-
-    if not more:
-      break
+  result = {
+      'hasMore': has_more,
+      'items': items,
+      'page': page,
+      'pageSize': PAGE_SIZE,
+      'totalItems': total_items,
+      'totalPages': total_pages,
+  }
+  return result, params
 
 
 class Handler(base_handler_flask.Handler):
@@ -96,17 +105,25 @@ class Handler(base_handler_flask.Handler):
   @handler_flask.check_user_access(need_privileged_access=False)
   def get(self):
     """Render the bot list HTML."""
-    if utils.is_oss_fuzz():
-      heartbeats = _get_host_workers_heartbeats()
-    else:
-      heartbeats = ndb_utils.get_all_from_model(data_types.Heartbeat)
-
-    bots = _convert_heartbeats_to_dicts(heartbeats)
+    result, params = get_results()
     return self.render('bots.html', {
-        'bots': bots,
+        'result': result,
+        'params': params,
     })
 
 
+class JsonHandler(base_handler_flask.Handler):
+  """Handler that gets the bots when user clicks on next page."""
+
+  @handler_flask.post(handler_flask.JSON, handler_flask.JSON)
+  @handler_flask.check_admin_access_if_oss_fuzz
+  @handler_flask.check_user_access(need_privileged_access=False)
+  def post(self):
+    """Get and render the bots in JSON."""
+    result, _ = get_results()
+    return self.render_json(result)
+
+
 class DeadBotsHandler(base_handler_flask.Handler):
   """Output dead bots as json."""
 

src/appengine/index.yaml

@@ -555,3 +555,8 @@ indexes:
   properties:
     - name: keywords
     - name: name
+
+- kind: Heartbeat
+  properties:
+    - name: keywords
+    - name: bot_name