auto extension #93

HISTORY.rst

@@ -12,6 +12,7 @@ Release History
 - Notebooks extensions can be prefixed with any prefix of at most three chars (#87)
 - Export of the same notebook to multiple formats is now supported. To export to all python formats, plus ``.ipynb`` and ``.md``, use ``"jupytext": {"formats": "ipynb,pct.py:percent,lgt.py:light,spx.py:sphinx,md"},``.
 - README includes a short section on how to extend ``light`` and ``percent`` formats to more languages (#61)
+- Jupytext's contents manager accepts the ``auto`` extension in ``default_jupytext_formats`` (#93)
 
 **BugFixes**
 

README.md

@@ -108,7 +108,7 @@ To enable paired notebooks, one option is to set the output formats by adding a
   }
 }
 ```
-Accepted formats are composed of an extension, like `ipynb`, `md`, `Rmd`, `jl`, `py`, `R`, and an optional format name among `light` (default for Julia, Python), `percent`, `sphinx`, `spin` (default for R). Use `ipynb,py:percent` if you want to pair the `.ipynb` notebook with a `.py` script in the `percent` format.
+Accepted formats are composed of an extension, like `ipynb`, `md`, `Rmd`, `jl`, `py`, `R`, and an optional format name among `light` (default for Julia, Python), `percent`, `sphinx`, `spin` (default for R). Use `ipynb,py:percent` if you want to pair the `.ipynb` notebook with a `.py` script in the `percent` format. To have the script extension chosen according to the Jupyter kernel, use the `auto` extension.
 
 Alternatively, it is also possible to set a default format pairing. Say you want to always associate `.ipynb` notebooks with an `.md` file  (and reciprocally). This is simply done by adding the following to your Jupyter configuration file:
 ```python

jupytext/contentsmanager.py

@@ -56,6 +56,8 @@ def check_formats(formats):
                        "Groups can be separated with colon, for instance: "
                        "'ipynb,nb.py;script.ipynb,py'")
 
+    allowed_extension = NOTEBOOK_EXTENSIONS + ['auto']
+
     validated_formats = []
     for group in formats:
         if not isinstance(group, list):
@@ -70,21 +72,15 @@ def check_formats(formats):
             try:
                 fmt = u'' + fmt
             except UnicodeDecodeError:
-                raise ValueError('Extensions should be strings among {}'
-                                 ', not {}.\n{}'
-                                 .format(str(formats.NOTEBOOK_EXTENSIONS),
-                                         str(fmt),
-                                         expected_format))
+                raise ValueError('Extensions should be strings among {}, not {}.\n{}'
+                                 .format(str(allowed_extension), str(fmt), expected_format))
             if fmt == '':
                 continue
             fmt, _ = parse_one_format(fmt)
-            if not any([fmt.endswith(ext)
-                        for ext in NOTEBOOK_EXTENSIONS]):
+            if not any([fmt.endswith(ext) for ext in allowed_extension]):
                 raise ValueError('Group extension {} contains {}, '
                                  'which does not end with either {}.\n{}'
-                                 .format(str(group), fmt,
-                                         str(NOTEBOOK_EXTENSIONS),
-                                         expected_format))
+                                 .format(str(group), fmt, str(allowed_extension), expected_format))
             if fmt.endswith('.ipynb'):
                 has_ipynb = True
             else:
@@ -170,6 +166,17 @@ def all_nb_extensions(self):
              'the ipynb notebook',
         config=True)
 
+    def replace_auto_ext(self, group, auto_ext):
+        """Replace any .auto extension with the given extension, and if none,
+        removes that alternative format from the group"""
+        result = []
+        for fmt in group:
+            if not fmt.endswith('.auto'):
+                result.append(fmt)
+            elif auto_ext:
+                result.append(fmt.replace('.auto', auto_ext))
+        return result
+
     def format_group(self, fmt, nbk=None):
         """Return the group of extensions that contains 'fmt'"""
         if nbk:
@@ -183,10 +190,15 @@ def format_group(self, fmt, nbk=None):
         except ValueError as err:
             raise HTTPError(400, str(err))
 
+        auto_ext = nbk.metadata.get('language_info', {}).get('file_extension') if nbk else None
+        if auto_ext == '.r':
+            auto_ext = '.R'
         # Find group that contains the current format
         for group in jupytext_formats:
+            if auto_ext and fmt.replace(auto_ext, '.auto') in group:
+                return self.replace_auto_ext(group, auto_ext)
             if fmt in group:
-                return group
+                return self.replace_auto_ext(group, auto_ext)
 
         # No such group, but 'ipynb'? Return current fmt + 'ipynb'
         if ['.ipynb'] in jupytext_formats:
@@ -196,10 +208,18 @@ def format_group(self, fmt, nbk=None):
 
     def preferred_format(self, ext, preferred):
         """Returns the preferred format for that extension"""
-        for fmt_ext, format_name in \
-                parse_formats(preferred):
+        for fmt_ext, format_name in parse_formats(preferred):
             if fmt_ext == ext:
                 return format_name
+            if not (ext.endswith('.md') or ext.endswith('.Rmd')):
+                if fmt_ext == '.auto':
+                    return format_name
+                if fmt_ext.endswith('.auto'):
+                    base_ext, ext_ext = os.path.splitext(ext)
+                    base_fmt, _ = os.path.splitext(fmt_ext)
+                    if base_ext == base_fmt and ext_ext:
+                        return format_name
+
         return None
 
     def _read_notebook(self, os_path, as_version=4):

jupytext/formats.py

@@ -137,11 +137,26 @@ def get_format(ext, format_name=None):
     raise TypeError("Not format associated to extension '{}'".format(ext))
 
 
-def read_format_from_metadata(text, ext):
-    """Return the format of the file, when that information is available from the metadata"""
+def read_metadata(text, ext):
+    """Return the header metadata"""
+    ext = '.' + ext.split('.')[-1]
     lines = text.splitlines()
 
-    metadata, _, _ = header_to_metadata_and_cell(lines, "#'" if ext == '.R' else '#')
+    if ext in ['.md', '.Rmd']:
+        comment = ''
+    else:
+        comment = _SCRIPT_EXTENSIONS.get(ext, {}).get('comment', '#')
+
+    metadata, _, _ = header_to_metadata_and_cell(lines, comment)
+    if ext == '.R' and not metadata:
+        metadata, _, _ = header_to_metadata_and_cell(lines, "#'")
+
+    return metadata
+
+
+def read_format_from_metadata(text, ext):
+    """Return the format of the file, when that information is available from the metadata"""
+    metadata = read_metadata(text, ext)
 
     transition_to_jupytext_section_in_metadata(metadata, ext.endswith('.ipynb'))
 
@@ -160,8 +175,7 @@ def guess_format(text, ext):
     """Guess the format of the file, given its extension and content"""
     lines = text.splitlines()
 
-    metadata, _, _ = header_to_metadata_and_cell(
-        lines, "#'" if ext == '.R' else '#')
+    metadata = read_metadata(text, ext)
 
     if ('jupytext' in metadata and set(metadata['jupytext']).difference(['encoding', 'main_language'])) or \
             set(metadata).difference(['jupytext']):

tests/test_contentsmanager.py

@@ -391,3 +391,64 @@ def test_preferred_format_allows_to_read_implicit_light_format(nb_file, tmpdir):
 
     # check that format (missing) is recognized as light
     assert 'py:light' in model['content']['metadata']['jupytext']['formats']
+
+
+@pytest.mark.parametrize('nb_file', list_notebooks('ipynb'))
+def test_save_in_auto_extension(nb_file, tmpdir):
+    # load notebook
+    nb = jupytext.readf(nb_file)
+    if 'language_info' not in nb.metadata:
+        return
+
+    auto_ext = nb.metadata['language_info']['file_extension']
+    if auto_ext == '.r':
+        auto_ext = '.R'
+    tmp_ipynb = 'notebook.ipynb'
+    tmp_script = 'notebook' + auto_ext
+
+    # create contents manager with default load format as percent
+    cm = jupytext.TextFileContentsManager()
+    cm.default_jupytext_formats = 'ipynb,auto'
+    cm.preferred_jupytext_formats_save = 'auto:percent'
+    cm.root_dir = str(tmpdir)
+
+    # save notebook
+    with mock.patch('jupytext.header.INSERT_AND_CHECK_VERSION_NUMBER', True):
+        cm.save(model=dict(type='notebook', content=nb), path=tmp_ipynb)
+
+    # check that text representation exists, and is in percent format
+    with open(str(tmpdir.join(tmp_script))) as stream:
+        assert read_format_from_metadata(stream.read(), auto_ext) == 'percent'
+
+
+@pytest.mark.parametrize('nb_file', list_notebooks('ipynb'))
+def test_save_in_pct_and_lgt_auto_extensions(nb_file, tmpdir):
+    # load notebook
+    nb = jupytext.readf(nb_file)
+    if 'language_info' not in nb.metadata:
+        return
+
+    auto_ext = nb.metadata['language_info']['file_extension']
+    if auto_ext == '.r':
+        auto_ext = '.R'
+    tmp_ipynb = 'notebook.ipynb'
+    tmp_pct_script = 'notebook.pct' + auto_ext
+    tmp_lgt_script = 'notebook.lgt' + auto_ext
+
+    # create contents manager with default load format as percent
+    cm = jupytext.TextFileContentsManager()
+    cm.default_jupytext_formats = 'ipynb,pct.auto,lgt.auto'
+    cm.preferred_jupytext_formats_save = 'pct.auto:percent,lgt.auto:light'
+    cm.root_dir = str(tmpdir)
+
+    # save notebook
+    with mock.patch('jupytext.header.INSERT_AND_CHECK_VERSION_NUMBER', True):
+        cm.save(model=dict(type='notebook', content=nb), path=tmp_ipynb)
+
+    # check that text representation exists in percent format
+    with open(str(tmpdir.join(tmp_pct_script))) as stream:
+        assert read_format_from_metadata(stream.read(), '.pct' + auto_ext) == 'percent'
+
+    # check that text representation exists in light format
+    with open(str(tmpdir.join(tmp_lgt_script))) as stream:
+        assert read_format_from_metadata(stream.read(), '.lgt' + auto_ext) == 'light'