diff --git a/HISTORY.rst b/HISTORY.rst index efba9548c..68c7d3a9a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -12,6 +12,7 @@ Release History - Notebooks extensions can be prefixed with any prefix of at most three chars (#87) - Export of the same notebook to multiple formats is now supported. To export to all python formats, plus ``.ipynb`` and ``.md``, use ``"jupytext": {"formats": "ipynb,pct.py:percent,lgt.py:light,spx.py:sphinx,md"},``. - README includes a short section on how to extend ``light`` and ``percent`` formats to more languages (#61) +- Jupytext's contents manager accepts the ``auto`` extension in ``default_jupytext_formats`` (#93) **BugFixes** diff --git a/README.md b/README.md index 0a111ed1b..2b5d3ee90 100755 --- a/README.md +++ b/README.md @@ -108,7 +108,7 @@ To enable paired notebooks, one option is to set the output formats by adding a } } ``` -Accepted formats are composed of an extension, like `ipynb`, `md`, `Rmd`, `jl`, `py`, `R`, and an optional format name among `light` (default for Julia, Python), `percent`, `sphinx`, `spin` (default for R). Use `ipynb,py:percent` if you want to pair the `.ipynb` notebook with a `.py` script in the `percent` format. +Accepted formats are composed of an extension, like `ipynb`, `md`, `Rmd`, `jl`, `py`, `R`, and an optional format name among `light` (default for Julia, Python), `percent`, `sphinx`, `spin` (default for R). Use `ipynb,py:percent` if you want to pair the `.ipynb` notebook with a `.py` script in the `percent` format. To have the script extension chosen according to the Jupyter kernel, use the `auto` extension. Alternatively, it is also possible to set a default format pairing. Say you want to always associate `.ipynb` notebooks with an `.md` file (and reciprocally). This is simply done by adding the following to your Jupyter configuration file: ```python diff --git a/jupytext/contentsmanager.py b/jupytext/contentsmanager.py index 59ee66547..d0ebc38bb 100644 --- a/jupytext/contentsmanager.py +++ b/jupytext/contentsmanager.py @@ -56,6 +56,8 @@ def check_formats(formats): "Groups can be separated with colon, for instance: " "'ipynb,nb.py;script.ipynb,py'") + allowed_extension = NOTEBOOK_EXTENSIONS + ['auto'] + validated_formats = [] for group in formats: if not isinstance(group, list): @@ -70,21 +72,15 @@ def check_formats(formats): try: fmt = u'' + fmt except UnicodeDecodeError: - raise ValueError('Extensions should be strings among {}' - ', not {}.\n{}' - .format(str(formats.NOTEBOOK_EXTENSIONS), - str(fmt), - expected_format)) + raise ValueError('Extensions should be strings among {}, not {}.\n{}' + .format(str(allowed_extension), str(fmt), expected_format)) if fmt == '': continue fmt, _ = parse_one_format(fmt) - if not any([fmt.endswith(ext) - for ext in NOTEBOOK_EXTENSIONS]): + if not any([fmt.endswith(ext) for ext in allowed_extension]): raise ValueError('Group extension {} contains {}, ' 'which does not end with either {}.\n{}' - .format(str(group), fmt, - str(NOTEBOOK_EXTENSIONS), - expected_format)) + .format(str(group), fmt, str(allowed_extension), expected_format)) if fmt.endswith('.ipynb'): has_ipynb = True else: @@ -170,6 +166,17 @@ def all_nb_extensions(self): 'the ipynb notebook', config=True) + def replace_auto_ext(self, group, auto_ext): + """Replace any .auto extension with the given extension, and if none, + removes that alternative format from the group""" + result = [] + for fmt in group: + if not fmt.endswith('.auto'): + result.append(fmt) + elif auto_ext: + result.append(fmt.replace('.auto', auto_ext)) + return result + def format_group(self, fmt, nbk=None): """Return the group of extensions that contains 'fmt'""" if nbk: @@ -183,10 +190,15 @@ def format_group(self, fmt, nbk=None): except ValueError as err: raise HTTPError(400, str(err)) + auto_ext = nbk.metadata.get('language_info', {}).get('file_extension') if nbk else None + if auto_ext == '.r': + auto_ext = '.R' # Find group that contains the current format for group in jupytext_formats: + if auto_ext and fmt.replace(auto_ext, '.auto') in group: + return self.replace_auto_ext(group, auto_ext) if fmt in group: - return group + return self.replace_auto_ext(group, auto_ext) # No such group, but 'ipynb'? Return current fmt + 'ipynb' if ['.ipynb'] in jupytext_formats: @@ -196,10 +208,18 @@ def format_group(self, fmt, nbk=None): def preferred_format(self, ext, preferred): """Returns the preferred format for that extension""" - for fmt_ext, format_name in \ - parse_formats(preferred): + for fmt_ext, format_name in parse_formats(preferred): if fmt_ext == ext: return format_name + if not (ext.endswith('.md') or ext.endswith('.Rmd')): + if fmt_ext == '.auto': + return format_name + if fmt_ext.endswith('.auto'): + base_ext, ext_ext = os.path.splitext(ext) + base_fmt, _ = os.path.splitext(fmt_ext) + if base_ext == base_fmt and ext_ext: + return format_name + return None def _read_notebook(self, os_path, as_version=4): diff --git a/jupytext/formats.py b/jupytext/formats.py index c982ac8a3..c3afaf940 100644 --- a/jupytext/formats.py +++ b/jupytext/formats.py @@ -137,11 +137,26 @@ def get_format(ext, format_name=None): raise TypeError("Not format associated to extension '{}'".format(ext)) -def read_format_from_metadata(text, ext): - """Return the format of the file, when that information is available from the metadata""" +def read_metadata(text, ext): + """Return the header metadata""" + ext = '.' + ext.split('.')[-1] lines = text.splitlines() - metadata, _, _ = header_to_metadata_and_cell(lines, "#'" if ext == '.R' else '#') + if ext in ['.md', '.Rmd']: + comment = '' + else: + comment = _SCRIPT_EXTENSIONS.get(ext, {}).get('comment', '#') + + metadata, _, _ = header_to_metadata_and_cell(lines, comment) + if ext == '.R' and not metadata: + metadata, _, _ = header_to_metadata_and_cell(lines, "#'") + + return metadata + + +def read_format_from_metadata(text, ext): + """Return the format of the file, when that information is available from the metadata""" + metadata = read_metadata(text, ext) transition_to_jupytext_section_in_metadata(metadata, ext.endswith('.ipynb')) @@ -160,8 +175,7 @@ def guess_format(text, ext): """Guess the format of the file, given its extension and content""" lines = text.splitlines() - metadata, _, _ = header_to_metadata_and_cell( - lines, "#'" if ext == '.R' else '#') + metadata = read_metadata(text, ext) if ('jupytext' in metadata and set(metadata['jupytext']).difference(['encoding', 'main_language'])) or \ set(metadata).difference(['jupytext']): diff --git a/tests/test_contentsmanager.py b/tests/test_contentsmanager.py index 63f9dd2bd..d9a4f4620 100644 --- a/tests/test_contentsmanager.py +++ b/tests/test_contentsmanager.py @@ -391,3 +391,64 @@ def test_preferred_format_allows_to_read_implicit_light_format(nb_file, tmpdir): # check that format (missing) is recognized as light assert 'py:light' in model['content']['metadata']['jupytext']['formats'] + + +@pytest.mark.parametrize('nb_file', list_notebooks('ipynb')) +def test_save_in_auto_extension(nb_file, tmpdir): + # load notebook + nb = jupytext.readf(nb_file) + if 'language_info' not in nb.metadata: + return + + auto_ext = nb.metadata['language_info']['file_extension'] + if auto_ext == '.r': + auto_ext = '.R' + tmp_ipynb = 'notebook.ipynb' + tmp_script = 'notebook' + auto_ext + + # create contents manager with default load format as percent + cm = jupytext.TextFileContentsManager() + cm.default_jupytext_formats = 'ipynb,auto' + cm.preferred_jupytext_formats_save = 'auto:percent' + cm.root_dir = str(tmpdir) + + # save notebook + with mock.patch('jupytext.header.INSERT_AND_CHECK_VERSION_NUMBER', True): + cm.save(model=dict(type='notebook', content=nb), path=tmp_ipynb) + + # check that text representation exists, and is in percent format + with open(str(tmpdir.join(tmp_script))) as stream: + assert read_format_from_metadata(stream.read(), auto_ext) == 'percent' + + +@pytest.mark.parametrize('nb_file', list_notebooks('ipynb')) +def test_save_in_pct_and_lgt_auto_extensions(nb_file, tmpdir): + # load notebook + nb = jupytext.readf(nb_file) + if 'language_info' not in nb.metadata: + return + + auto_ext = nb.metadata['language_info']['file_extension'] + if auto_ext == '.r': + auto_ext = '.R' + tmp_ipynb = 'notebook.ipynb' + tmp_pct_script = 'notebook.pct' + auto_ext + tmp_lgt_script = 'notebook.lgt' + auto_ext + + # create contents manager with default load format as percent + cm = jupytext.TextFileContentsManager() + cm.default_jupytext_formats = 'ipynb,pct.auto,lgt.auto' + cm.preferred_jupytext_formats_save = 'pct.auto:percent,lgt.auto:light' + cm.root_dir = str(tmpdir) + + # save notebook + with mock.patch('jupytext.header.INSERT_AND_CHECK_VERSION_NUMBER', True): + cm.save(model=dict(type='notebook', content=nb), path=tmp_ipynb) + + # check that text representation exists in percent format + with open(str(tmpdir.join(tmp_pct_script))) as stream: + assert read_format_from_metadata(stream.read(), '.pct' + auto_ext) == 'percent' + + # check that text representation exists in light format + with open(str(tmpdir.join(tmp_lgt_script))) as stream: + assert read_format_from_metadata(stream.read(), '.lgt' + auto_ext) == 'light'