Merge tag '19.0.1' into nii-mergework-201907

fujimoto · fujimoto · commit ddcf87b42de5 · 2019-07-08T17:14:02.000+09:00
diff --git a/CHANGELOG b/CHANGELOG
@@ -2,6 +2,24 @@
 ChangeLog
 *********
 
+19.0.1 (2019-06-20)
+===================
+- Fix: Update Dockerfile to install LibreOffice v6.1.6.  v6.1.5 is no longer available from their
+archive.
+
+19.0.0 (2019-06-19)
+===================
+- Fix: Add white background to transparent .gifs to improve zooming appearance.  The zooming code
+keeps a copy of the image behind the zoomed version to improve performance.  If the image is
+transparent the actual-size copy will be visible behind the zoomed version.  Making the background
+opaque hides the original.
+- Feature: Track when the hypothes.is sidebar is opened via Google Analytics.  (thanks, @adlius!)
+- Feature: Add timeouts to renderers that shell out to external processes for conversion.  If a
+conversion takes longer than the configured number of seconds, MFR will return an error message
+instead of waiting forever.  (thanks, @ExProbitasFiducia!)
+- Code: Remove Rackspace cache-management tasks.  MFR is now hosted on GoogleCloud, and these are no
+longer needed.  (thanks, @ExProbitasFiducia!)
+
 18.0.0 (2018-10-11)
 ===================
 - UPDATE: MFR is now following the CalVer (https://calver.org/) versioning scheme to match the
diff --git a/mfr/extensions/image/static/js/jquery.zoom.js b/mfr/extensions/image/static/js/jquery.zoom.js
@@ -4,10 +4,11 @@
  * Original Copy: https://github.com/jackmoore/zoom/blob/1.7.20/jquery.zoom.js
  * Version: https://github.com/jackmoore/zoom/releases/tag/1.7.20
  *
- * The are two MFR customizations in this file, one for style and one for functionality
+ * The are three MFR customizations in this file, two for style and one for functionality
  *
  * 1. Updated code style according to `.eslintrc.js`
  * 2. Added `.on("mousewheel", function (e) { ... })` to enable further zoom by mouse wheel scroll
+ * 3. Set "background-color: white" for "zoomImage" to handle images with transparent background
  */
 
 (function ($) {
@@ -53,6 +54,7 @@
                 top: 0,
                 left: 0,
                 opacity: 0,
+                "background-color": "white",
                 width: img.width * magnify,
                 height: img.height * magnify,
                 border: "none",
diff --git a/mfr/extensions/jsc3d/export.py b/mfr/extensions/jsc3d/export.py
@@ -1,13 +1,17 @@
-import os
 import shutil
-import subprocess
 from http import HTTPStatus
+from subprocess import (DEVNULL,
+                        check_call,
+                        TimeoutExpired,
+                        CalledProcessError)
+from os.path import basename, splitext
 from tempfile import NamedTemporaryFile
 
 from mfr.core import exceptions
 from mfr.core.extension import BaseExporter
-from mfr.extensions.jsc3d.settings import (FREECAD_BIN,
-                                           FREECAD_CONVERT_SCRIPT)
+from mfr.extensions.jsc3d.settings import (TIMEOUT,
+                                           FREECAD_BIN,
+                                           CONVERSION_SCRIPT)
 
 
 class JSC3DExporter(BaseExporter):
@@ -22,16 +26,14 @@ def export(self):
                 temp_source_file.name = self.source_file_path + '.step'
                 shutil.copy2(self.source_file_path, temp_source_file.name)
 
-                subprocess.check_call([
-                    FREECAD_BIN,
-                    FREECAD_CONVERT_SCRIPT,
-                    temp_source_file.name,
-                    self.output_file_path,
-                    # silence output from freecadcmd
-                ], stdout=subprocess.DEVNULL)
+                check_call(
+                    [FREECAD_BIN, CONVERSION_SCRIPT, temp_source_file.name, self.output_file_path],
+                    stdout=DEVNULL,  # silence output from freecadcmd
+                    timeout=TIMEOUT,
+                )
 
-        except subprocess.CalledProcessError as err:
-            name, extension = os.path.splitext(os.path.split(self.source_file_path)[-1])
+        except CalledProcessError as err:
+            name, extension = splitext(basename(self.source_file_path))
             raise exceptions.SubprocessError(
                 'Unable to export the file in the requested format, please try again later.',
                 process='freecad',
@@ -42,3 +44,19 @@ def export(self):
                 extension=extension or '',
                 exporter_class='jsc3d',
             )
+
+        except TimeoutExpired as err:
+            name, extension = splitext(basename(self.source_file_path))
+            # The return code 52 is not the error code returned by the
+            # subprocess, but the error given to it by this waterbutler
+            # processs, for timing out.
+            raise exceptions.SubprocessError(
+                'JSC3D Conversion timed out.',
+                code=HTTPStatus.GATEWAY_TIMEOUT,
+                process='freecad',
+                cmd=str(err.cmd),
+                returncode=52,
+                path=str(self.source_file_path),
+                extension=extension or '',
+                exporter_class='jsc3d'
+            )
diff --git a/mfr/extensions/jsc3d/settings.py b/mfr/extensions/jsc3d/settings.py
@@ -3,6 +3,7 @@
 config = settings.child('JSC3D_EXTENSION_CONFIG')
 
 FREECAD_BIN = config.get('FREECAD_BIN', '/usr/bin/freecadcmd')
-FREECAD_CONVERT_SCRIPT = config.get('FREECAD_CONVERT_SCRIPT', '/code/mfr/extensions/jsc3d/freecad_converter.py')
+CONVERSION_SCRIPT = config.get('FREECAD_CONVERT_SCRIPT', '/code/mfr/extensions/jsc3d/freecad_converter.py')
+TIMEOUT = int(config.get('FREECAD_TIMEOUT', 30))  # In seconds
 EXPORT_TYPE = config.get('EXPORT_TYPE', '.stl')
 EXPORT_EXCLUSIONS = config.get('EXPORT_EXCLUSIONS', '.3ds .stl .obj .ctm').split(' ')
diff --git a/mfr/extensions/pdf/render.py b/mfr/extensions/pdf/render.py
@@ -7,6 +7,7 @@
 from mfr.core import extension
 from mfr.extensions.pdf import settings
 from mfr.extensions.utils import munge_url_for_localdev, escape_url_for_template
+from mfr.settings import GOOGLE_ANALYTICS_TRACKING_ID
 
 logger = logging.getLogger(__name__)
 
@@ -29,6 +30,7 @@ def render(self):
         if self.metadata.ext.lower() not in settings.EXPORT_SUPPORTED:
             logger.debug('Extension not found in supported list!')
             return self.TEMPLATE.render(
+                ga_tracking_id=GOOGLE_ANALYTICS_TRACKING_ID,
                 base=self.assets_url,
                 url=escape_url_for_template(download_url.geturl()),
                 stable_id=self.metadata.stable_id,
@@ -46,6 +48,7 @@ def render(self):
 
         self.metrics.add('needs_export', True)
         return self.TEMPLATE.render(
+            ga_tracking_id=GOOGLE_ANALYTICS_TRACKING_ID,
             base=self.assets_url,
             url=escape_url_for_template(exported_url.url),
             stable_id=self.metadata.stable_id,
diff --git a/mfr/extensions/pdf/templates/viewer.mako b/mfr/extensions/pdf/templates/viewer.mako
@@ -426,6 +426,12 @@ http://sourceforge.net/adobe/cmap/wiki/License/
     <script>
         window.MFR_STABLE_ID = '${stable_id}';
         window.MFR_FILE_NAME = '${file_name}';
+        window.GA_TRACKING_ID = '${ga_tracking_id}';
+        (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+        (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+        m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+        })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
+        ga('create', window.GA_TRACKING_ID, 'auto');
     </script>
     <script src="/static/js/mfr.child.hypothesis.js"></script>
     % endif
diff --git a/mfr/extensions/tabular/settings.py b/mfr/extensions/tabular/settings.py
@@ -37,3 +37,4 @@
 })
 
 PSPP_CONVERT_BIN = config.get('PSPP_CONVERT_BIN', '/usr/bin/pspp-convert')
+PSPP_CONVERT_TIMEOUT = int(config.get('PSPP_CONVERT_TIMEOUT', 30))  # In seconds
diff --git a/mfr/extensions/tabular/utilities.py b/mfr/extensions/tabular/utilities.py
@@ -1,9 +1,14 @@
 import re
-import subprocess
+from http import HTTPStatus
+from subprocess import (check_call,
+                        TimeoutExpired,
+                        CalledProcessError)
 from tempfile import NamedTemporaryFile
 
-from mfr.core import exceptions
-from mfr.extensions.tabular import compat, settings
+from mfr.extensions.tabular import compat
+from mfr.core.exceptions import SubprocessError
+from mfr.extensions.tabular.settings import (PSPP_CONVERT_BIN,
+                                             PSPP_CONVERT_TIMEOUT)
 
 
 def header_population(headers):
@@ -48,20 +53,33 @@ def sav_to_csv(fp):
     """
     csv_file = NamedTemporaryFile(mode='w+b', suffix='.csv')
     try:
-        subprocess.check_call([
-            settings.PSPP_CONVERT_BIN,
-            fp.name,
-            csv_file.name,
-        ])
-    except subprocess.CalledProcessError as err:
-        raise exceptions.SubprocessError(
+        check_call(
+            [PSPP_CONVERT_BIN, fp.name, csv_file.name],
+            timeout=PSPP_CONVERT_TIMEOUT,
+        )
+    except CalledProcessError as err:
+        raise SubprocessError(
             'Unable to convert the SPSS file to CSV, please try again later.',
-            code=500,
+            code=HTTPStatus.INTERNAL_SERVER_ERROR,
             process='pspp',
             cmd=str(err.cmd),
             returncode=err.returncode,
             path=fp.name,
             extension='sav',
             exporter_class='tabular',
         )
+    except TimeoutExpired as err:
+        # The return code 52 is not the error code returned by the
+        # subprocess, but the error given to it by this waterbutler
+        # processs, for timing out.
+        raise SubprocessError(
+            'CSV Conversion timed out.',
+            code=HTTPStatus.GATEWAY_TIMEOUT,
+            process='pspp',
+            cmd=str(err.cmd),
+            returncode=52,
+            path=fp.name,
+            extension='sav',
+            exporter_class='tabular'
+        )
     return csv_file
diff --git a/mfr/extensions/unoconv/export.py b/mfr/extensions/unoconv/export.py
@@ -1,35 +1,37 @@
-import os
-import subprocess
+from http import HTTPStatus
+from os.path import basename, splitext
+from subprocess import run, CalledProcessError
 
-from mfr.core import extension
-from mfr.core import exceptions
+from mfr.core.extension import BaseExporter
+from mfr.core.exceptions import SubprocessError
+from mfr.extensions.unoconv.settings import (PORT,
+                                             ADDRESS,
+                                             UNOCONV_BIN,
+                                             UNOCONV_TIMEOUT)
 
-from mfr.extensions.unoconv import settings
 
-
-class UnoconvExporter(extension.BaseExporter):
+class UnoconvExporter(BaseExporter):
 
     def export(self):
         try:
-            subprocess.run([
-                settings.UNOCONV_BIN,
+            run([
+                UNOCONV_BIN,
                 '-n',
-                '-c', 'socket,host={},port={};urp;StarOffice.ComponentContext'.format(settings.ADDRESS, settings.PORT),
+                '-c', 'socket,host={},port={};urp;StarOffice.ComponentContext'.format(ADDRESS, PORT),
                 '-f', self.format,
                 '-o', self.output_file_path,
                 '-vvv',
                 self.source_file_path
-            ], check=True, timeout=settings.UNOCONV_TIMEOUT)
-
-        except subprocess.CalledProcessError as err:
-            name, extension = os.path.splitext(os.path.split(self.source_file_path)[-1])
-            raise exceptions.SubprocessError(
+            ], check=True, timeout=UNOCONV_TIMEOUT)
+        except CalledProcessError as err:
+            name, extension = splitext(basename(self.source_file_path))
+            raise SubprocessError(
                 'Unable to export the file in the requested format, please try again later.',
                 process='unoconv',
                 cmd=str(err.cmd),
                 returncode=err.returncode,
                 path=str(self.source_file_path),
-                code=400,
+                code=HTTPStatus.BAD_REQUEST,
                 extension=extension or '',
                 exporter_class='unoconv',
             )
diff --git a/mfr/server/static/js/mfr.child.hypothesis.js b/mfr/server/static/js/mfr.child.hypothesis.js
@@ -62,7 +62,34 @@
             window.document.head.appendChild(script);
             window.document.body.classList.add('show-hypothesis');
             hypothesisLoaded = true;
-        });
 
+            var sidePanelOpened = false;
+            // window.DEFAULT_URL should be the wb link in this format:
+            // https://<wb-domain>/v1/resources/<preprint-guid>/providers/osfstorage/<file-id>?direct=&mode=render
+            // TODO: parse and validate the WB URL before retrieving the preprints GUID
+            var wbLink = window.DEFAULT_URL;
+            var preprintGuid;
+            if (wbLink.split('/').length >= 6) {
+                preprintGuid = wbLink.split('/')[5];
+            } else {
+                preprintGuid = 'preprint-guid-unknown';
+            }
+            var sendAnalyticsIfExpanded = function (expanded) { 
+                if (expanded && !sidePanelOpened) {
+                    sidePanelOpened = expanded;
+                    ga('send', 'event', {
+                        eventCategory: 'Hypothesis',
+                        eventAction: 'Open Hypothesis Panel',
+                        //`eventLabel` is the guid of the preprint to which the file belongs
+                        eventLabel: preprintGuid,
+                    });
+                }
+            };
+            window.hypothesisConfig = function () {
+                return {
+                  "onLayoutChange": function (layout) { return sendAnalyticsIfExpanded(layout.expanded); }
+                };
+            };
+        });
     };
 })();
diff --git a/mfr/settings.py b/mfr/settings.py
@@ -170,3 +170,4 @@ def child(key):
 
 
 SENTRY_DSN = config.get_nullable('SENTRY_DSN', None)
+GOOGLE_ANALYTICS_TRACKING_ID = config.get_nullable('GOOGLE_ANALYTICS_TRACKING_ID', None)
diff --git a/mfr/version.py b/mfr/version.py
@@ -1 +1 @@
-__version__ = '18.0.0'
+__version__ = '19.0.1'
diff --git a/tasks.py b/tasks.py
@@ -85,71 +85,3 @@ def server(ctx):
 
     from mfr.server.app import serve
     serve()
-
-@task
-def export_cache_clean(ctx, extension=''):
-    cache_clean('export/', extension)
-
-@task
-def render_cache_clean(ctx, extension=''):
-    cache_clean('render/', extension)
-
-def cache_clean(folder, extension):
-    # NOTE: Manually install gevent & pyrax, no need for it to be depenency just for this method.
-
-    from gevent import monkey
-    from gevent.pool import Pool
-    from gevent import Timeout
-
-    monkey.patch_all()
-
-    import six
-    import pyrax
-    import logging
-    from mfr.server import settings
-
-    # Monkey patch pyrax for python 3 compatibility.
-    def _add_details(self, info):
-        """
-        Takes the dict returned by the API call and sets the
-        corresponding attributes on the object.
-        """
-        for (key, val) in six.iteritems(info):
-            if six.PY2 and isinstance(key, six.text_type):
-                key = key.encode(pyrax.get_encoding())
-            elif isinstance(key, bytes):
-                key = key.decode("utf-8")
-            setattr(self, key, val)
-    pyrax.resource.BaseResource._add_details = _add_details
-
-    # WARNING: We are using provider specific functions to enumerate files to quickly
-    # purge the cache, which can contain hundreds of thousands of objects. Thus
-    # asserting the provider, we will need to update if we move providers.
-    assert settings.CACHE_PROVIDER_NAME == 'cloudfiles'
-
-    logging.captureWarnings(True)
-
-    pyrax.set_setting('identity_type', 'rackspace')
-    pyrax.set_setting('verify_ssl', True)
-    pyrax.set_credentials(settings.CACHE_PROVIDER_CREDENTIALS['username'], settings.CACHE_PROVIDER_CREDENTIALS['token'])
-
-    cf = pyrax.connect_to_cloudfiles(region=settings.CACHE_PROVIDER_CREDENTIALS['region'].upper(), public=True)
-    container = cf.get_container(settings.CACHE_PROVIDER_SETTINGS['container'])
-
-    def delete_object(obj):
-        # added timeout of 5 seconds just in case
-        with Timeout(5, False):
-            try:
-                print(obj)
-                obj.delete()
-            except Exception as ex:
-                print(ex)
-
-    pool = Pool(100)
-    objects = container.get_objects(prefix=folder, limit=5000, marker='')
-    while objects:
-        for obj in objects:
-            if obj.name.endswith(extension):
-                pool.spawn(delete_object, obj)
-        objects = container.get_objects(prefix=folder, limit=5000, marker=objects[-1].name)
-    pool.join()

Original file line number	Diff line number	Diff line change
`@@ -37,3 +37,4 @@`
`37`	`37`	`})`
`38`	`38`
`39`	`39`	`PSPP_CONVERT_BIN = config.get('PSPP_CONVERT_BIN', '/usr/bin/pspp-convert')`
	`40`	`+PSPP_CONVERT_TIMEOUT = int(config.get('PSPP_CONVERT_TIMEOUT', 30)) # In seconds`