Merge pull request #72 from mwouts/v0.6.4

V0.6.4

HISTORY.rst

@@ -3,8 +3,18 @@
 Release History
 ---------------
 
-dev
-+++
+0.6.4 (2018-09-12)
++++++++++++++++++++
+
+**Improvements**
+
+- Jupytext will not load paired notebook when text representation is out of date (#63)
+- Package tested against Python 3.7 (#68)
+
+**BugFixes**
+
+- Allow unicode characters in notebook path (#70)
+- Read README.md as unicode in `setup.py` (#71)
 
 0.6.3 (2018-09-07)
 +++++++++++++++++++

binder/requirements.txt

@@ -1,4 +1,4 @@
-jupytext>=0.6.3
+jupytext>=0.6.4
 plotly
 matplotlib
 pandas

jupytext/combine.py

@@ -25,6 +25,7 @@ def combine_inputs_with_outputs(nb_source, nb_outputs):
 
                 metadata = ocell.metadata
                 cell.metadata.update({k: metadata[k] for k in metadata
-                                      if metadata in _IGNORE_METADATA})
+                                      if k == 'trusted' or
+                                      k not in _IGNORE_METADATA})
                 remaining_output_cells = remaining_output_cells[(i + 1):]
                 break

jupytext/contentsmanager.py

@@ -1,20 +1,21 @@
 """ContentsManager that allows to open Rmd, py, R and ipynb files as notebooks
 """
 import os
+from datetime import timedelta
 import nbformat
 import mock
-import six
+from tornado.web import HTTPError
+from traitlets import Unicode, Float
+from traitlets.config import Configurable
+
+from notebook.services.contents.filemanager import FileContentsManager
 
 try:
     import notebook.transutils  # noqa
 except ImportError:
     pass
-from notebook.services.contents.filemanager import FileContentsManager
-from tornado.web import HTTPError
-from traitlets import Unicode
-from traitlets.config import Configurable
-import jupytext
 
+import jupytext
 from . import combine
 from .file_format_version import check_file_version
 
@@ -56,9 +57,14 @@ def check_formats(formats):
             # In early versions (0.4 and below), formats could be a list of
             # extensions. We understand this as a single group
             return check_formats([formats])
+
+        # Return ipynb on first position (save that file first, for #63)
+        has_ipynb = False
         validated_group = []
         for fmt in group:
-            if not isinstance(fmt, six.string_types):
+            try:
+                fmt = u'' + fmt
+            except UnicodeDecodeError:
                 raise ValueError('Extensions should be strings among {}'
                                  ', not {}.\n{}'
                                  .format(str(jupytext.NOTEBOOK_EXTENSIONS),
@@ -75,7 +81,13 @@ def check_formats(formats):
                                  .format(str(group), fmt,
                                          str(jupytext.NOTEBOOK_EXTENSIONS),
                                          expected_format))
-            validated_group.append(fmt)
+            if fmt == '.ipynb':
+                has_ipynb = True
+            else:
+                validated_group.append(fmt)
+
+        if has_ipynb:
+            validated_group = ['.ipynb'] + validated_group
 
         if validated_group:
             validated_formats.append(validated_group)
@@ -116,8 +128,21 @@ def all_nb_extensions(self):
              'comma separated',
         config=True)
 
+    outdated_text_notebook_margin = Float(
+        1.0,
+        help='Refuse to overwrite inputs of a ipynb notebooks with those of a'
+             'text notebook when the text notebook plus margin is older than'
+             'the ipynb notebook',
+        config=True)
+
     def format_group(self, fmt, nbk=None):
         """Return the group of extensions that contains 'fmt'"""
+        # Backward compatibility with nbrmd
+        for key in ['nbrmd_formats', 'nbrmd_format_version']:
+            if nbk and key in nbk.metadata:
+                nbk.metadata[key.replace('nbrmd', 'jupytext')] = \
+                    nbk.metadata.pop(key)
+
         jupytext_formats = ((nbk.metadata.get('jupytext_formats')
                              if nbk else None) or
                             self.default_jupytext_formats)
@@ -132,78 +157,23 @@ def format_group(self, fmt, nbk=None):
             if fmt in group:
                 return group
 
+        # No such group, but 'ipynb'? Return current fmt + 'ipynb'
         if ['.ipynb'] in jupytext_formats:
-            return [fmt, '.ipynb']
+            return ['.ipynb', fmt]
 
         return [fmt]
 
-    def _read_notebook(self, os_path, as_version=4,
-                       load_alternative_format=True):
+    def _read_notebook(self, os_path, as_version=4):
         """Read a notebook from an os path."""
-        file, fmt, ext = file_fmt_ext(os_path)
+        _, ext = os.path.splitext(os_path)
         if ext in self.nb_extensions:
             with mock.patch('nbformat.reads', _jupytext_reads(ext)):
-                nbk = super(TextFileContentsManager, self) \
+                return super(TextFileContentsManager, self) \
                     ._read_notebook(os_path, as_version)
         else:
-            nbk = super(TextFileContentsManager, self) \
+            return super(TextFileContentsManager, self) \
                 ._read_notebook(os_path, as_version)
 
-        if not load_alternative_format:
-            return nbk
-
-        fmt_group = self.format_group(fmt, nbk)
-
-        source_format = fmt
-        outputs_format = fmt
-
-        # Source format is first non ipynb format found on disk
-        if fmt.endswith('.ipynb'):
-            for alt_fmt in fmt_group:
-                if not alt_fmt.endswith('.ipynb') and \
-                        os.path.isfile(file + alt_fmt):
-                    source_format = alt_fmt
-                    break
-        # Outputs taken from ipynb if in group, if file exists
-        else:
-            for alt_fmt in fmt_group:
-                if alt_fmt.endswith('.ipynb') and \
-                        os.path.isfile(file + alt_fmt):
-                    outputs_format = alt_fmt
-                    break
-
-        if source_format != fmt:
-            self.log.info('Reading SOURCE from {}'
-                          .format(os.path.basename(file + source_format)))
-            nb_outputs = nbk
-            nbk = self._read_notebook(file + source_format,
-                                      as_version=as_version,
-                                      load_alternative_format=False)
-        elif outputs_format != fmt:
-            self.log.info('Reading OUTPUTS from {}'
-                          .format(os.path.basename(file + outputs_format)))
-            if outputs_format != fmt:
-                nb_outputs = self._read_notebook(file + outputs_format,
-                                                 as_version=as_version,
-                                                 load_alternative_format=False)
-        else:
-            nb_outputs = None
-
-        try:
-            check_file_version(nbk, file + source_format,
-                               file + outputs_format)
-        except ValueError as err:
-            raise HTTPError(400, str(err))
-
-        if nb_outputs:
-            combine.combine_inputs_with_outputs(nbk, nb_outputs)
-            if self.notary.check_signature(nb_outputs):
-                self.notary.sign(nbk)
-        elif not fmt.endswith('.ipynb'):
-            self.notary.sign(nbk)
-
-        return nbk
-
     def _save_notebook(self, os_path, nb):
         """Save a notebook to an os_path."""
         os_file, fmt, _ = file_fmt_ext(os_path)
@@ -219,16 +189,103 @@ def _save_notebook(self, os_path, nb):
                 super(TextFileContentsManager, self) \
                     ._save_notebook(os_path_fmt, nb)
 
-    def get(self, path, content=True, type=None, format=None):
+    def get(self, path, content=True, type=None, format=None,
+            load_alternative_format=True):
         """ Takes a path for an entity and returns its model"""
         path = path.strip('/')
+        nb_file, fmt, ext = file_fmt_ext(path)
 
         if self.exists(path) and \
                 (type == 'notebook' or
-                 (type is None and
-                  any([path.endswith(ext)
-                       for ext in self.all_nb_extensions()]))):
-            return self._notebook_model(path, content=content)
+                 (type is None and ext in self.all_nb_extensions())):
+            model = self._notebook_model(path, content=content)
+            if not content:
+                return model
+
+            if not load_alternative_format:
+                return model
+
+            fmt_group = self.format_group(fmt, model['content'])
+
+            source_format = fmt
+            outputs_format = fmt
+
+            # Source format is first non ipynb format found on disk
+            if fmt.endswith('.ipynb'):
+                for alt_fmt in fmt_group:
+                    if not alt_fmt.endswith('.ipynb') and \
+                            self.exists(nb_file + alt_fmt):
+                        source_format = alt_fmt
+                        break
+            # Outputs taken from ipynb if in group, if file exists
+            else:
+                for alt_fmt in fmt_group:
+                    if alt_fmt.endswith('.ipynb') and \
+                            self.exists(nb_file + alt_fmt):
+                        outputs_format = alt_fmt
+                        break
+
+            if source_format != fmt:
+                self.log.info(u'Reading SOURCE from {}'.format(
+                    os.path.basename(nb_file + source_format)))
+                model_outputs = model
+                model = self.get(nb_file + source_format, content=content,
+                                 type=type, format=format,
+                                 load_alternative_format=False)
+            elif outputs_format != fmt:
+                self.log.info(u'Reading OUTPUTS from {}'.format(
+                    os.path.basename(nb_file + outputs_format)))
+                model_outputs = self.get(nb_file + outputs_format,
+                                         content=content,
+                                         type=type, format=format,
+                                         load_alternative_format=False)
+            else:
+                model_outputs = None
+
+            try:
+                check_file_version(model['content'],
+                                   nb_file + source_format,
+                                   nb_file + outputs_format)
+            except ValueError as err:
+                raise HTTPError(400, str(err))
+
+            # Make sure we're not overwriting ipynb cells with an outdated
+            # text file
+            try:
+                if model_outputs and model_outputs['last_modified'] > \
+                        model['last_modified'] + \
+                        timedelta(seconds=self.outdated_text_notebook_margin):
+                    raise HTTPError(
+                        400,
+                        u'\n'
+                        '{out} (last modified {out_last})\n'
+                        'seems more recent than '
+                        '{src} (last modified {src_last})\n'
+                        'Please either:\n'
+                        '- open {src} in a text editor, make sure it is '
+                        'up to date, and save it,\n'
+                        '- or delete {src} if not up to date,\n'
+                        '- or increase check margin by adding, say, \n'
+                        '    c.ContentsManager.'
+                        'outdated_text_notebook_margin = 5 '
+                        '# in seconds # or float("inf")\n'
+                        'to your .jupyter/jupyter_notebook_config.py '
+                        'file\n'.format(src=nb_file + source_format,
+                                        src_last=model['last_modified'],
+                                        out=nb_file + outputs_format,
+                                        out_last=model_outputs[
+                                            'last_modified']))
+            except OverflowError:
+                pass
+
+            if model_outputs:
+                combine.combine_inputs_with_outputs(model['content'],
+                                                    model_outputs['content'])
+            elif not fmt.endswith('.ipynb'):
+                self.notary.sign(model['content'])
+                self.mark_trusted_cells(model['content'], path)
+
+            return model
 
         return super(TextFileContentsManager, self) \
             .get(path, content, type, format)

jupytext/file_format_version.py

@@ -58,7 +58,7 @@ def check_file_version(notebook, source_path, outputs_path):
         return
 
     # Version larger than minimum readable version
-    if version <= current and version >= min_file_format_version(ext):
+    if min_file_format_version(ext) <= version <= current:
         return
 
         # Not merging? OK

jupytext/jupytext.py

@@ -77,12 +77,6 @@ def to_notebook(self, text):
 
         set_main_and_cell_language(metadata, cells, self.ext)
 
-        # Backward compatibility with nbrmd
-        for key in ['nbrmd_formats', 'nbrmd_format_version']:
-            if key in metadata:
-                metadata[key.replace('nbrmd', 'jupytext')] = \
-                    metadata.pop(key)
-
         return new_notebook(cells=cells, metadata=metadata)
 
 

requirements.txt

@@ -1,5 +1,4 @@
 nbformat>=4.0.0
 pyyaml
 mock
-six
 testfixtures

setup.py

@@ -1,13 +1,14 @@
 from os import path
+from io import open
 from setuptools import setup, find_packages
 
 this_directory = path.abspath(path.dirname(__file__))
-with open(path.join(this_directory, 'README.md')) as f:
+with open(path.join(this_directory, 'README.md'), encoding='utf-8') as f:
     long_description = f.read()
 
 setup(
     name='jupytext',
-    version='0.6.3',
+    version='0.6.4',
     author='Marc Wouts',
     author_email='marc.wouts@gmail.com',
     description='Jupyter notebooks as Markdown documents, '
@@ -22,7 +23,7 @@
                        'pynotebook = jupytext:PyNotebookExporter',
                        'rnotebook = jupytext:RNotebookExporter']},
     tests_require=['pytest'],
-    install_requires=['nbformat>=4.0.0', 'mock', 'pyyaml', 'six', 'testfixtures'],
+    install_requires=['nbformat>=4.0.0', 'mock', 'pyyaml', 'testfixtures'],
     license='MIT',
     classifiers=('Development Status :: 4 - Beta',
                  'Environment :: Console',

tests/test_cli.py

@@ -175,5 +175,5 @@ def test_combine_lower_version_raises(tmpdir):
                         {'.py': '1.0'}):
             with mock.patch(
                     'jupytext.file_format_version.MIN_FILE_FORMAT_VERSION',
-                            {'.py': '1.0'}):
+                    {'.py': '1.0'}):
                 jupytext(args=[tmp_nbpy, '--to', 'ipynb', '--update'])