Improved install (#94)

* Requirements file for AWE Workbench

* Non-working commit / fixing requirements to work with setup.py / still need to fix -e

* Subdirectories work in req.txt

* awe_requirements

* Missing init file. Why did this ever work?

* lo_dash_react_components, hashes in setup.py

* lo_dash_react_components now works after one install

* Writing observer modules

* Module has standard place for 3rd party libs

* wo_common_student_errors copies assets to virtualenv

* Dash integration is using pkg_resources, albeit not completely correctly

* We now handle pip installed modules with assets

* Reasonable error handling in dash_integration.py

* Better error on roster data

* Better auth error message

* Update to supporting latest bootstrap

* Writing Observer requirements in format usable from github

* Extra requires for writing observer. Might not be stable.

* added other common assets to downloads and addressed runtime to json encoding error

* Fixed typo in module.py

---------

Co-authored-by: Piotr Mitros <piotr@mitros.org>

Author: bradley-erickson

awe_requirements.txt

@@ -0,0 +1,5 @@
+AWE_SpellCorrect @ git+https://github.com/ETS-Next-Gen/AWE_SpellCorrect.git
+AWE_Components @ git+https://github.com/ETS-Next-Gen/AWE_Components.git
+AWE_Lexica @ git+https://github.com/ETS-Next-Gen/AWE_Lexica.git
+AWE_LanguageTool @ git+https://github.com/ETS-Next-Gen/AWE_LanguageTool.git
+AWE_Workbench @ git+https://github.com/ETS-Next-Gen/AWE_Workbench.git

gitserve/gitserve/__init__.py

@@ -5,18 +5,16 @@
 Views for monitoring overall system operation, and eventually, for
 administering the system.
 '''
-import copy
-import numbers
 import psutil
 import sys
 
 import aiohttp
 import aiohttp.web
 
-import dash.development.base_component
 
 import learning_observer.module_loader
 from learning_observer.log_event import debug_log
+from learning_observer.util import clean_json
 
 from learning_observer.auth.utils import admin
 
@@ -72,34 +70,6 @@ def routes(app):
                 resources.append(sinfo)
         return resources
 
-    def clean_json(json_object):
-        '''
-        * Deep copy a JSON object
-        * Convert list-like objects to lists
-        * Convert dictionary-like objects to dicts
-        * Convert functions to string representations
-        '''
-        if isinstance(json_object, str):
-            return str(json_object)
-        if isinstance(json_object, numbers.Number):
-            return json_object
-        if isinstance(json_object, dict):
-            return {key: clean_json(value) for key, value in json_object.items()}
-        if isinstance(json_object, list) or isinstance(json_object, tuple):
-            return [clean_json(i) for i in json_object]
-        if isinstance(json_object, learning_observer.stream_analytics.fields.Scope):
-            # We could make a nicer representation....
-            return str(json_object)
-        if callable(json_object):
-            return str(json_object)
-        if json_object is None:
-            return json_object
-        if str(type(json_object)) == "<class 'module'>":
-            return str(json_object)
-        if isinstance(json_object, dash.development.base_component.Component):
-            return f"Dash Component {json_object}"
-        raise ValueError("We don't yet handle this type in clean_json: {} (object: {})".format(type(json_object), json_object))
-
     status = {
         "status": "Alive!",
         "resources": machine_resources(),

learning_observer/learning_observer/admin.py

@@ -106,5 +106,6 @@ def verify_auth_precheck():
                 "mirrors the one here.\n" + \
                 "\n" + \
                 "If you are not planning to use Google auth (which is the case for most dev\n" + \
-                "settings), please disable Google authentication in creds.yaml"
+                "settings), please disable Google authentication in creds.yaml by\n" + \
+                "removing the google_auth section under auth."
             raise learning_observer.prestartup.StartupCheck(error)

learning_observer/learning_observer/auth/__init__.py

@@ -17,7 +17,7 @@
 import learning_observer.settings
 import learning_observer.stream_analytics.fields
 import learning_observer.stream_analytics.helpers
-from learning_observer.util import get_nested_dict_value
+from learning_observer.util import get_nested_dict_value, clean_json
 from learning_observer.communication_protocol.exception import DAGExecutionException
 
 dispatch = learning_observer.communication_protocol.query.dispatch
@@ -647,10 +647,10 @@ async def visit(node_name):
 
     # Include execution history in output if operating in development settings
     if learning_observer.settings.RUN_MODE == learning_observer.settings.RUN_MODES.DEV:
-        return {e: await visit(e) for e in target_nodes}
+        return {e: clean_json(await visit(e)) for e in target_nodes}
 
     # Remove execution history if in deployed settings, with data flowing back to teacher dashboards
-    return {e: strip_provenance(await visit(e)) for e in target_nodes}
+    return {e: clean_json(strip_provenance(await visit(e))) for e in target_nodes}
 
 
 if __name__ == "__main__":

learning_observer/learning_observer/communication_protocol/executor.py

@@ -13,6 +13,9 @@
 import os.path
 import shutil
 
+from pkg_resources import resource_filename, resource_listdir
+
+
 import dash
 from dash import Dash, html, clientside_callback, Output, Input
 
@@ -121,7 +124,19 @@ def asset_paths():
             module = modules[m]
             for layout in module:
                 if 'ASSETS' in layout:
-                    asset_path = os.path.join(layout['_BASE_PATH'], layout['ASSETS'])
+                    try:
+                        # This is a hack. We should be using resource_dir and then resource_filename.
+                        asset_path = resource_filename(m, layout['ASSETS'])
+                    except TypeError:
+                        print(
+                            f'\n\n'
+                            f'If you are seeing an odd exception in posixpath.py, then module {m}\n'
+                            'is probably not set up correctly. It is defining assets in module.py,\n'
+                            'but I cannot find them. It is probably either missing __init__.py in\n'
+                            'a directory, a MANIFEST.in for static assets, or a package_data field\n'
+                            'in setup.py (or the equivalent).\n\n')
+                        raise
+                    # OLD: os.path.join(layout['_BASE_PATH'], layout['ASSETS'])
                     yield asset_path
 
     def copy_files(source, destination):

learning_observer/learning_observer/dash_integration.py

@@ -0,0 +1,134 @@
+'''This file contains default versions of components.
+
+This is handy for:
+- Making changes in one place
+- Keeping everyone on the same version
+
+URL is where we grab the component from. Note that DBC components
+change frequently.
+
+Hashes allow us to verify file integrity / avoid man-in-the-middle
+attacks
+
+The name is currently not used, but it'd be handy to be able to
+include components and component sets as lists (rather than
+dictionaries) and with common names.
+
+tested_versions is also currently not used (although a few modules
+have a hacked-up version of something similar. The idea is we should
+be following LATEST in dev, but on servers, flag if we're running a
+version which we have not yet tested.
+'''
+
+import dash_bootstrap_components as dbc
+import os
+
+REQUIRE_JS = {
+    "url": "https://requirejs.org/docs/release/2.3.6/comments/require.js",
+    "hash": "d1e7687c1b2990966131bc25a761f03d6de83115512c9ce85d72e4b9819fb"
+    "8733463fa0d93ca31e2e42ebee6e425d811e3420a788a0fc95f745aa349e3b01901",
+    "name": "require.js"  # <-- We should switch to something like this
+}
+TEXT_JS = {
+    "url": "https://raw.githubusercontent.com/requirejs/text/"
+    "3f9d4c19b3a1a3c6f35650c5788cbea1db93197a/text.js",
+    "hash": "fb8974f1633f261f77220329c7070ff214241ebd33a1434f2738572608efc"
+    "8eb6699961734285e9500bbbd60990794883981fb113319503208822e6706bca0b8"
+}
+R_JS = {
+    "url": "https://requirejs.org/docs/release/2.3.6/r.js",
+    "hash": "52300a8371df306f45e981fd224b10cc586365d5637a19a24e710a2fa566f"
+    "88450b8a3920e7af47ba7197ffefa707a179bc82a407f05c08508248e6b5084f457"
+}
+BULMA_CSS = {
+    "url": "https://cdnjs.cloudflare.com/ajax/libs/bulma/0.9.0/css/"
+    "bulma.min.css",
+    "hash": "ec7342883fdb6fbd4db80d7b44938951c3903d2132fc3e4bf7363c6e6dc52"
+    "95a478c930856177ac6257c32e1d1e10a4132c6c51d194b3174dc670ab8d116b362"
+}
+FONTAWESOME_JS = {
+    "url": "https://use.fontawesome.com/releases/v5.3.1/js/all.js",
+    "hash": "83e7b36f1545d5abe63bea9cd3505596998aea272dd05dee624b9a2c72f96"
+    "62618d4bff6e51fafa25d41cb59bd97f3ebd72fd94ebd09a52c17c4c23fdca3962b"
+}
+SHOWDOWN_JS = {
+    "url": "https://rawgit.com/showdownjs/showdown/1.9.1/dist/showdown.js",
+    "hash": "4fe14f17c2a1d0275d44e06d7e68d2b177779196c6d0c562d082eb5435eec"
+    "4e710a625be524767aef3d9a1f6a5b88f912ddd71821f4a9df12ff7dd66d6fbb3c9"
+}
+SHOWDOWN_JS_MAP = {
+    "url": "https://rawgit.com/showdownjs/showdown/1.9.1/dist/showdown.js.map",
+    "hash": "74690aa3cea07fd075942ba9e98cf7297752994b93930acb3a1baa2d3042a"
+    "62b5523d3da83177f63e6c02fe2a09c8414af9e1774dad892a303e15a86dbeb29ba"
+}
+MUSTACHE_JS = {
+    "url": "http://cdnjs.cloudflare.com/ajax/libs/mustache.js/3.1.0/"
+    "mustache.min.js",
+    "hash": "e7c446dc9ac2da9396cf401774efd9bd063d25920343eaed7bee9ad878840"
+    "e846d48204d62755aede6f51ae6f169dcc9455f45c1b86ba1b42980ccf8f241af25"
+}
+D3_V5_JS = {
+    "url": "https://d3js.org/d3.v5.min.js",
+    "hash": "466fe57816d719048885357cccc91a082d8e5d3796f227f88a988bf36a5c2"
+    "ceb7a4d25842f5f3c327a0151d682e648cd9623bfdcc7a18a70ac05cfd0ec434463"
+}
+BULMA_TOOLTIP_CSS = {
+    "url": "https://cdn.jsdelivr.net/npm/@creativebulma/bulma-tooltip@1.2.0/"
+    "dist/bulma-tooltip.min.css",
+    "hash": "fc37b25fa75664a6aa91627a7b1298a09025c136085f99ba31b1861f073a0"
+    "696c4756cb156531ccf5c630154d66f3059b6b589617bd6bd711ef665079f879405"
+}
+BOOTSTRAP_MIN_CSS  = {
+    "url": dbc.themes.MINTY,
+    "hash": {
+        "old": "b361dc857ee7c817afa9c3370f1d317db2c4be5572dd5ec3171caeb812281"
+        "cf900a5a9141e5d6c7069408e2615df612fbcd31094223996154e16f2f80a348532",
+        "5.1.3": "c03f5bfd8deb11ad6cec84a6201f4327f28a640e693e56466fd80d983ed54"
+        "16deff1548a0f6bbad013ec278b9750d1d253bd9c5bd1f53c85fcd62adba5eedc59",
+        "5.3.1": "d099dac0135309466dc6208aaa973584843a3efbb40b2c96eb7c179f5f20f"
+        "80def35bbc1a7a0b08c9d5bdbed6b8e780ba7d013d18e4019e04fd82a19c076a1f8"
+    },
+    "tested_versions": [
+        "https://cdn.jsdelivr.net/npm/bootswatch@5.3.1/dist/minty/bootstrap.min.css",
+        'https://cdn.jsdelivr.net/npm/bootswatch@5.1.3/dist/minty/bootstrap.min.css',
+    ]
+}
+FONTAWESOME_CSS = {
+    "url": dbc.icons.FONT_AWESOME,
+    "hash": {
+        "6.1.1": "535a5f3e40bc8ddf475b56c1a39a5406052b524413dea331c4e683ca99e39"
+        "6dbbc11fdce1f8355730a73c52ac6a1062de1938406c6af8e4361fd346106acb6b0",
+        "6.3.0": "1496214e7421773324f4b332127ea77bec822fc6739292ebb19c6abcc22a5"
+        "6248e0634b4e0ca0c2fcac14dc10b8d01fa17febaa35f46731201d1ffd0ab482dd7"
+    },
+    "tested_versions": [
+        "https://use.fontawesome.com/releases/v6.3.0/css/all.css",
+        "https://use.fontawesome.com/releases/v6.1.1/css/all.css"
+    ]
+}
+FONTAWESOME_WOFF2 = {
+    "url": os.path.dirname(os.path.dirname(dbc.icons.FONT_AWESOME)) + "/webfonts/fa-solid-900.woff2",
+    "hash": {
+        "6.1.1": "6d3fe769cc40a5790ea2e09fb775f1bd3b130d2fdae1dd552f69559e7ca4c"
+        "a047862f795da0024737e59e3bcc7446f6eec1bab173758aef0b97ba89d722ffbde",
+        "6.3.0": "d50c68cd4b3312f50deb66ac8ab5c37b2d4161f4e00ea077"
+        "326ae76769dac650dd19e65dee8d698ba2f86a69537f38cf4010ff45227211cee8b382d9b567257a"
+    }
+}
+FONTAWESOME_TTF = {
+    "url": os.path.dirname(os.path.dirname(dbc.icons.FONT_AWESOME)) + "/webfonts/fa-solid-900.ttf",
+    "hash": {
+        "6.1.1": "0fdd341671021d04304186c197001cf2e888d3028baaf9a5dec0f0e496959"
+        "666e8a2e34aae8e79904f8e9b4c0ccae40249897cce5f5ae58d12cc1b3985e588d6",
+        "6.3.0": "5a2c2b010a2496e4ed832ede8620f3bbfa9374778f3d63e4"
+        "5a4aab041e174dafd9fffd3229b8b36f259cf2ef46ae7bf5cb041e280f2939884652788fc1e8ce58"
+    }
+}
+
+if (dbc.themes.MINTY not in BOOTSTRAP_MIN_CSS["tested_versions"]):
+    print("WARN:: Unrecognized Minty URL detected: {}".format(dbc.themes.MINTY))
+    print("You will need to update dash bootstrap components hash value.\n")
+
+if (dbc.icons.FONT_AWESOME not in FONTAWESOME_CSS["tested_versions"]):
+    print("WARN:: Unrecognized Fontawesome URL detected: {}".format(dbc.icons.FONT_AWESOME))
+    print("You will need to update the FontAwesome bootstrap components hash value.\n")

learning_observer/learning_observer/downloads.py

@@ -374,11 +374,13 @@ def init():
                 raise learning_observer.prestartup.StartupCheck(
                     "Missing course roster files!\n"
                     "The following are required:\t{paths}\n\n"
-                    "Please run:\n"
+                    "If you plan to use rosters from local files, please run:\n"
                     "{commands}\n\n"
                     "(And ideally, they'll be populated with\n"
                     "a list of courses, and of students for\n"
-                    "those courses)".format(
+                    "those courses)\n\n"
+                    "If you plan to use other sources of roster,\n"
+                    "data please change creds.yaml instead".format(
                         paths=", ".join(r_paths),
                         commands="\n".join(["mkdir {path}".format(path=path) for path in r_paths])
                     )

learning_observer/learning_observer/rosters.py

@@ -9,14 +9,18 @@
 before doing so.
 '''
 
+import dash.development.base_component
 import datetime
 import enum
 import hashlib
 import math
+import numbers
 import re
 import socket
 from dateutil import parser
 
+import learning_observer
+
 
 def paginate(data_list, nrows):
     '''
@@ -152,6 +156,39 @@ def get_nested_dict_value(d, key_str=None, default=MissingType.Missing):
     return d
 
 
+def clean_json(json_object):
+    '''
+    * Deep copy a JSON object
+    * Convert list-like objects to lists
+    * Convert dictionary-like objects to dicts
+    * Convert functions to string representations
+    '''
+    if isinstance(json_object, str):
+        return str(json_object)
+    if isinstance(json_object, numbers.Number):
+        return json_object
+    if isinstance(json_object, dict):
+        return {key: clean_json(value) for key, value in json_object.items()}
+    if isinstance(json_object, list) or isinstance(json_object, tuple):
+        return [clean_json(i) for i in json_object]
+    if isinstance(json_object, learning_observer.stream_analytics.fields.Scope):
+        # We could make a nicer representation....
+        return str(json_object)
+    if callable(json_object):
+        return str(json_object)
+    if json_object is None:
+        return json_object
+    if str(type(json_object)) == "<class 'module'>":
+        return str(json_object)
+    if str(type(json_object)) == "<class 'learning_observer.runtime.Runtime'>":
+        return str(json_object)
+    if isinstance(json_object, dash.development.base_component.Component):
+        return f"Dash Component {json_object}"
+    if isinstance(json_object, KeyError):
+        return str(json_object)
+    raise ValueError("We don't yet handle this type in clean_json: {} (object: {})".format(type(json_object), json_object))
+
+
 def timestamp():
     """
     Return a timestamp string in ISO 8601 format

learning_observer/learning_observer/util.py

@@ -6,6 +6,24 @@
 '''
 
 from setuptools import setup
+import os
+
+my_path = os.path.dirname(os.path.realpath(__file__))
+parent_path = os.path.abspath(os.path.join(my_path, os.pardir))
+req_path = os.path.join(parent_path, 'requirements.txt')
+awe_path = os.path.join(parent_path, 'awe_requirements.txt')
+wo_path = os.path.join(parent_path, 'wo_requirements.txt')
+
+def clean_requirements(filename):
+    file_path = os.path.join(parent_path, filename)
+    requirements = [s.strip() for s in open(file_path).readlines() if len(s)>1]
+    return requirements
+
 
 setup(
+    install_requires = clean_requirements(req_path),
+    extras_require = {
+        "wo": clean_requirements(wo_path),
+        "awe": clean_requirements(awe_path)
+    }
 )