From d7260c61bc30b80ab256330c3aa9d60814d1e840 Mon Sep 17 00:00:00 2001 From: Marc Wouts Date: Mon, 29 Oct 2018 23:25:05 +0100 Subject: [PATCH] freeze_metadata option #110 - Changed `additional_metadata_on_text_files` to `freeze_metadata` - Option added to the command line --- README.md | 4 ++-- jupytext/cli.py | 14 ++++++++++---- jupytext/contentsmanager.py | 17 ++++++++--------- jupytext/jupytext.py | 28 ++++++++++++++++------------ tests/test_header.py | 8 +++++--- 5 files changed, 41 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 3718f4a66..7835ff15e 100755 --- a/README.md +++ b/README.md @@ -177,7 +177,7 @@ jupytext --test --update notebook.ipynb -to py:percent Note that `jupytext --test` compares the resulting notebooks according to its expectations. If you wish to proceed to a strict comparison of the two notebooks, use `jupytext --test-strict`, and use the flag `-x` to report with more details on the first difference, if any. Please note that -- When you associate a Jupyter kernel with your text notebook, that information goes to a YAML header at the top of your script or Markdown document. And Jupytext itself may create a `jupytext` entry in the notebook metadata. +- When you associate a Jupyter kernel with your text notebook, that information goes to a YAML header at the top of your script or Markdown document. And Jupytext itself may create a `jupytext` entry in the notebook metadata. Have a look at the [`freeze_metadata` option](#cell-and-notebook-metadata-filtering) if you want to avoid this. - Cell metadata are available in `light` and `percent` formats for all cell types. Sphinx Gallery scripts in `sphinx` format do not support cell metadata. R Markdown and R scripts in `spin` format support cell metadata for code cells only. Markdown documents do not support cell metadata. - By default, a few cell metadata are not included in the text representation of the notebook. And only the most standard notebook metadata are exported. Learn more on this in this in the [metadata filtering](#Cell-and-notebook-metadata-filtering) section. - Representing a Jupyter notebook as a Markdown or R Markdown document has the effect of splitting markdown cells with two consecutive blank lines into multiple cells (as the two blank line pattern is used to separate cells). @@ -295,7 +295,7 @@ Help us improving the default configuration: if you are aware of a notebook meta Finally, if you prefer that scripts and markdown files with no YAML header do not get one (nor additional cell metadata) when opened and saved in Jupyter, set the following option on Jupytext's content manager: ```python -c.ContentsManager.additional_metadata_on_text_files = False +c.ContentsManager.freeze_metadata = True ``` ## Extending the `light` and `percent` formats to more languages diff --git a/jupytext/cli.py b/jupytext/cli.py index 3ccd8296f..03d7b3008 100644 --- a/jupytext/cli.py +++ b/jupytext/cli.py @@ -14,7 +14,7 @@ def convert_notebook_files(nb_files, fmt, input_format=None, output=None, test_round_trip=False, test_round_trip_strict=False, stop_on_first_error=True, - update=True): + update=True, freeze_metadata=False): """ Export R markdown notebooks, python or R scripts, or Jupyter notebooks, to the opposite format @@ -49,7 +49,8 @@ def convert_notebook_files(nb_files, fmt, input_format=None, output=None, if nb_file == sys.stdin: dest = None current_ext, _ = parse_one_format(input_format) - notebook = reads(nb_file.read(), ext=current_ext, format_name=format_name) + notebook = reads(nb_file.read(), ext=current_ext, format_name=format_name, + freeze_metadata=freeze_metadata) else: dest, current_ext = os.path.splitext(nb_file) notebook = None @@ -67,7 +68,8 @@ def convert_notebook_files(nb_files, fmt, input_format=None, output=None, input_format = None if not notebook: - notebook = readf(nb_file, format_name=format_name) + notebook = readf(nb_file, format_name=format_name, + freeze_metadata=freeze_metadata) if test_round_trip or test_round_trip_strict: try: @@ -175,6 +177,9 @@ def cli_jupytext(args=None): parser.add_argument('--update', action='store_true', help='Preserve outputs of .ipynb destination ' '(when file exists and inputs match)') + parser.add_argument('--freeze-metadata', action='store_true', + help='Filter notebook and cell metadata that are not in the text notebook. ' + 'Use this to avoid creating a YAML header when editing text files.') test = parser.add_mutually_exclusive_group() test.add_argument('--test', dest='test', action='store_true', help='Test that notebook is stable under ' @@ -224,7 +229,8 @@ def jupytext(args=None): test_round_trip=args.test, test_round_trip_strict=args.test_strict, stop_on_first_error=args.stop_on_first_error, - update=args.update) + update=args.update, + freeze_metadata=args.freeze_metadata) except ValueError as err: # (ValueError, TypeError, IOError) as err: print('jupytext: error: ' + str(err)) exit(1) diff --git a/jupytext/contentsmanager.py b/jupytext/contentsmanager.py index 58a66573b..581637765 100644 --- a/jupytext/contentsmanager.py +++ b/jupytext/contentsmanager.py @@ -30,10 +30,10 @@ def _writes(nbk, version=nbformat.NO_CONVERT, **kwargs): return _writes -def _jupytext_reads(ext, format_name, rst2md, additional_metadata_on_text_files): +def _jupytext_reads(ext, format_name, rst2md, freeze_metadata): def _reads(text, as_version, **kwargs): return jupytext.reads(text, ext=ext, format_name=format_name, rst2md=rst2md, - additional_metadata_on_text_files=additional_metadata_on_text_files, + freeze_metadata=freeze_metadata, as_version=as_version, **kwargs) return _reads @@ -151,12 +151,11 @@ def all_nb_extensions(self): "Examples: 'all', 'hide_input,hide_output'", config=True) - additional_metadata_on_text_files = Bool( - True, - help='Allow (or not) additional notebook and cell metadata to be saved to a text file ' - 'that has no "jupyter" section or no YAML header.', - config=True - ) + freeze_metadata = Bool( + False, + help='Filter notebook and cell metadata that are not in the text notebook. ' + 'Use this to avoid creating a YAML header when editing text files.', + config=True) comment_magics = Enum( values=[True, False], @@ -240,7 +239,7 @@ def _read_notebook(self, os_path, as_version=4): format_name = self.preferred_format(fmt, self.preferred_jupytext_formats_read) with mock.patch('nbformat.reads', _jupytext_reads(fmt, format_name, self.sphinx_convert_rst2md, - self.additional_metadata_on_text_files)): + self.freeze_metadata)): return super(TextFileContentsManager, self)._read_notebook(os_path, as_version) else: return super(TextFileContentsManager, self)._read_notebook(os_path, as_version) diff --git a/jupytext/jupytext.py b/jupytext/jupytext.py index b4ce23181..8ff523779 100644 --- a/jupytext/jupytext.py +++ b/jupytext/jupytext.py @@ -26,10 +26,10 @@ class TextNotebookReader(NotebookReader): """Text notebook reader""" - def __init__(self, ext, format_name=None, additional_metadata_on_text_files=True): + def __init__(self, ext, format_name=None, freeze_metadata=False): self.ext = ext self.format = get_format(ext, format_name) - self.additional_metadata_on_text_files = additional_metadata_on_text_files + self.freeze_metadata = freeze_metadata def reads(self, s, **_): """Read a notebook from text""" @@ -58,10 +58,11 @@ def reads(self, s, **_): raise Exception('Blocked at lines ' + '\n'.join(lines[:6])) lines = lines[pos:] - if not self.additional_metadata_on_text_files and not metadata: - metadata['jupytext'] = {'metadata_filter': {'notebook': False}} - if not cell_metadata: - metadata['jupytext']['metadata_filter']['cells'] = False + if self.freeze_metadata: + metadata['jupytext'] = {'metadata_filter': { + 'notebook': {'additional': list(metadata.keys()), 'excluded': 'all'}}} + metadata['jupytext']['metadata_filter']['cells'] = { + 'additional': list(cell_metadata), 'excluded': 'all'} set_main_and_cell_language(metadata, cells, self.format.extension) @@ -139,7 +140,7 @@ def writes(self, nb, **kwargs): def reads(text, ext, format_name=None, - rst2md=False, additional_metadata_on_text_files=True, as_version=4, **kwargs): + rst2md=False, freeze_metadata=False, as_version=4, **kwargs): """Read a notebook from a string""" if ext.endswith('.ipynb'): return nbformat.reads(text, as_version, **kwargs) @@ -151,7 +152,7 @@ def reads(text, ext, format_name=None, if format_name == 'sphinx' and rst2md: format_name = 'sphinx-rst2md' - reader = TextNotebookReader(ext, format_name, additional_metadata_on_text_files) + reader = TextNotebookReader(ext, format_name, freeze_metadata) notebook = reader.reads(text, **kwargs) transition_to_jupytext_section_in_metadata(notebook.metadata, False) @@ -165,21 +166,24 @@ def reads(text, ext, format_name=None, return notebook -def read(file_or_stream, ext, format_name=None, as_version=4, **kwargs): +def read(file_or_stream, ext, format_name=None, + freeze_metadata=False, as_version=4, **kwargs): """Read a notebook from a file""" if ext.endswith('.ipynb'): notebook = nbformat.read(file_or_stream, as_version, **kwargs) transition_to_jupytext_section_in_metadata(notebook.metadata, True) return notebook - return reads(file_or_stream.read(), ext=ext, format_name=format_name, **kwargs) + return reads(file_or_stream.read(), ext=ext, format_name=format_name, + freeze_metadata=freeze_metadata, **kwargs) -def readf(nb_file, format_name=None): +def readf(nb_file, format_name=None, freeze_metadata=False): """Read a notebook from the file with given name""" _, ext = os.path.splitext(nb_file) with io.open(nb_file, encoding='utf-8') as stream: - return read(stream, as_version=4, ext=ext, format_name=format_name) + return read(stream, as_version=4, ext=ext, format_name=format_name, + freeze_metadata=freeze_metadata) def writes(notebook, ext, format_name=None, diff --git a/tests/test_header.py b/tests/test_header.py index 5e24d1305..fbb92aca3 100644 --- a/tests/test_header.py +++ b/tests/test_header.py @@ -94,9 +94,11 @@ def test_metadata_and_cell_to_header2(): def test_notebook_from_plain_script_has_metadata_filter(script="""print('Hello world") """): with mock.patch('jupytext.header.INSERT_AND_CHECK_VERSION_NUMBER', True): - nb = jupytext.reads(script, '.py', additional_metadata_on_text_files=False) - assert nb.metadata.get('jupytext', {}).get('metadata_filter', {}).get('notebook') is False - assert nb.metadata.get('jupytext', {}).get('metadata_filter', {}).get('cells') is False + nb = jupytext.reads(script, '.py', freeze_metadata=True) + assert nb.metadata.get('jupytext', {}).get('metadata_filter', {}).get('notebook') == { + 'additional': [], 'excluded': 'all'} + assert nb.metadata.get('jupytext', {}).get('metadata_filter', {}).get('cells') == { + 'additional': [], 'excluded': 'all'} with mock.patch('jupytext.header.INSERT_AND_CHECK_VERSION_NUMBER', True): scripts2 = jupytext.writes(nb, '.py')