ENH Clean-up code by early initialization of sphinx_gallery_conf (#1120)

Author: lesteve

sphinx_gallery/backreferences.py

@@ -350,8 +350,7 @@ def _finalize_backreferences(seen_backrefs, gallery_conf):
                 ex_file.write(THUMBNAIL_PARENT_DIV_CLOSE)
             _replace_md5(path, mode='t')
         else:
-            level = gallery_conf['log_level'].get('backreference_missing',
-                                                  'warning')
+            level = gallery_conf['log_level']['backreference_missing']
             func = getattr(logger, level)
             func(f'Could not find backreferences file: {path}')
             func('The backreferences are likely to be erroneous '

sphinx_gallery/gen_gallery.py

@@ -122,24 +122,6 @@ def _bool_eval(x):
     return bool(x)
 
 
-def parse_config(app, check_keys=True):
-    """Process the Sphinx Gallery configuration."""
-    plot_gallery = _bool_eval(app.builder.config.plot_gallery)
-    src_dir = app.builder.srcdir
-    abort_on_example_error = _bool_eval(
-        app.builder.config.abort_on_example_error)
-    lang = app.builder.config.highlight_language
-    gallery_conf = _complete_gallery_conf(
-        app.config.sphinx_gallery_conf, src_dir, plot_gallery,
-        abort_on_example_error, lang, app.builder.name, app,
-        check_keys)
-
-    # this assures I can call the config in other places
-    app.config.sphinx_gallery_conf = gallery_conf
-    app.config.html_static_path.append(glr_path_static())
-    return gallery_conf
-
-
 def _update_gallery_conf(gallery_conf):
     """Update gallery config.
 
@@ -151,19 +133,24 @@ def _update_gallery_conf(gallery_conf):
     gallery_conf['exclude_implicit_doc_regex'] = exclude_regex
 
 
-def _complete_gallery_conf(sphinx_gallery_conf, src_dir, plot_gallery,
-                           abort_on_example_error, lang='python',
-                           builder_name='html', app=None, check_keys=True):
+def _complete_gallery_conf_builder_inited(
+        sphinx_gallery_conf, src_dir, plot_gallery=True,
+        abort_on_example_error=False, lang='python',
+        builder_name='html'):
+    sphinx_gallery_conf.update(plot_gallery=plot_gallery)
+    sphinx_gallery_conf.update(abort_on_example_error=abort_on_example_error)
+    sphinx_gallery_conf['src_dir'] = src_dir
+    lang = lang if lang in ('python', 'python3', 'default') else 'python'
+    sphinx_gallery_conf['lang'] = lang
+    # Make it easy to know which builder we're in
+    sphinx_gallery_conf['builder_name'] = builder_name
+
+
+def _complete_gallery_conf_config_inited(sphinx_gallery_conf, app=None,
+                                         check_keys=True):
     gallery_conf = copy.deepcopy(DEFAULT_GALLERY_CONF)
     options = sorted(gallery_conf)
-    # sphinx-autoapi can run before our config filling can occur, which
-    # when used with show_api_usage=True can cause us to add this early.
-    # In theory we could bump our priority above theirs, but this could have
-    # unintented consequences.
-    ignore_keys = ('_sg_api_entries',)
-    extra_keys = sorted(
-        (set(sphinx_gallery_conf) - set(options)) - set(ignore_keys)
-    )
+    extra_keys = sorted(set(sphinx_gallery_conf) - set(options))
     if extra_keys and check_keys:
         msg = 'Unknown key(s) in sphinx_gallery_conf:\n'
         for key in extra_keys:
@@ -176,13 +163,10 @@ def _complete_gallery_conf(sphinx_gallery_conf, src_dir, plot_gallery,
             msg += '\n'
         raise ConfigError(msg.strip())
     gallery_conf.update(sphinx_gallery_conf)
-    gallery_conf.update(plot_gallery=plot_gallery)
-    gallery_conf.update(abort_on_example_error=abort_on_example_error)
     # XXX anything that can only be a bool (rather than str) should probably be
     # evaluated this way as it allows setting via -D on the command line
     for key in ('promote_jupyter_magic', 'run_stale_examples',):
         gallery_conf[key] = _bool_eval(gallery_conf[key])
-    gallery_conf['src_dir'] = src_dir
     gallery_conf['app'] = app
 
     # Check capture_repr
@@ -318,8 +302,6 @@ def call_memory(func):
                           "['before', 'after', 'both'], "
                           'got %r' % gallery_conf['reset_modules_order'])
 
-    lang = lang if lang in ('python', 'python3', 'default') else 'python'
-    gallery_conf['lang'] = lang
     del resetters
 
     # Ensure the first cell text is a string if we have it
@@ -357,8 +339,6 @@ def call_memory(func):
                                   "values: %s, got: %s."
                                   % (accepted_keys, key))
 
-    # Make it easy to know which builder we're in
-    gallery_conf['builder_name'] = builder_name
     gallery_conf['titles'] = {}
     # Ensure 'backreferences_dir' is str, pathlib.Path or None
     backref = gallery_conf['backreferences_dir']
@@ -376,7 +356,6 @@ def call_memory(func):
     gallery_conf['binder'] = check_binder_conf(gallery_conf['binder'])
 
     # jupyterlite
-    print('_complete_gallery_conf', gallery_conf.get('jupyterlite'))
     gallery_conf['jupyterlite'] = check_jupyterlite_conf(
         gallery_conf.get('jupyterlite', {}), app)
 
@@ -503,7 +482,7 @@ def generate_gallery_rst(app):
     """
 
     logger.info('generating gallery...', color='white')
-    gallery_conf = parse_config(app)
+    gallery_conf = app.config.sphinx_gallery_conf
 
     seen_backrefs = set()
 
@@ -737,9 +716,6 @@ def write_computation_times(gallery_conf, target_dir, costs):
 
 
 def write_api_entries(app, what, name, obj, options, lines):
-    # sphinx-autoapi can do this before we manage to complete our config
-    if 'show_api_usage' not in app.config.sphinx_gallery_conf:
-        app.config.sphinx_gallery_conf['show_api_usage'] = False
     if app.config.sphinx_gallery_conf['show_api_usage'] is False:
         return
     if '_sg_api_entries' not in app.config.sphinx_gallery_conf:
@@ -1065,7 +1041,7 @@ def _parse_failures(gallery_conf):
     # filter from examples actually run
     passing_unexpectedly = [
         src_file for src_file in passing_unexpectedly
-        if re.search(gallery_conf.get('filename_pattern'), src_file)]
+        if re.search(gallery_conf['filename_pattern'], src_file)]
     return failing_as_expected, failing_unexpectedly, passing_unexpectedly
 
 
@@ -1187,12 +1163,38 @@ def default_getter(conf):
     return default_getter
 
 
+def normalize_gallery_conf_config_inited(app, config, check_keys=True):
+    new_sphinx_gallery_conf = _complete_gallery_conf_config_inited(
+        config.sphinx_gallery_conf, app=app, check_keys=check_keys)
+    config.sphinx_gallery_conf = new_sphinx_gallery_conf
+    config.html_static_path.append(glr_path_static())
+
+
+def normalize_gallery_conf_builder_inited(app):
+    plot_gallery = _bool_eval(app.builder.config.plot_gallery)
+    src_dir = app.builder.srcdir
+    abort_on_example_error = _bool_eval(
+        app.builder.config.abort_on_example_error)
+    lang = app.builder.config.highlight_language
+    _complete_gallery_conf_builder_inited(
+        app.config.sphinx_gallery_conf,
+        src_dir,
+        plot_gallery=plot_gallery,
+        abort_on_example_error=abort_on_example_error,
+        lang=lang,
+        builder_name=app.builder.name
+    )
+
+
 def setup(app):
     """Setup Sphinx-Gallery sphinx extension"""
     app.add_config_value('sphinx_gallery_conf', DEFAULT_GALLERY_CONF, 'html')
     for key in ['plot_gallery', 'abort_on_example_error']:
         app.add_config_value(key, get_default_config_value(key), 'html')
 
+    # Early normalization of sphinx_gallery_conf at config-inited
+    app.connect('config-inited', normalize_gallery_conf_config_inited,
+                priority=10)
     # set small priority value, so that pre_configure_jupyterlite_sphinx is
     # called before jupyterlite_sphinx config-inited
     app.connect(
@@ -1213,6 +1215,9 @@ def setup(app):
 
     imagesg_addnode(app)
 
+    # Early normalization of sphinx_gallery_conf at builder-inited
+    app.connect('builder-inited', normalize_gallery_conf_builder_inited,
+                priority=10)
     app.connect('builder-inited', generate_gallery_rst)
     app.connect('build-finished', copy_binder_files)
     app.connect('build-finished', create_jupyterlite_contents)

sphinx_gallery/gen_rst.py

@@ -48,8 +48,8 @@
                                remove_ignore_blocks)
 
 from .notebook import jupyter_notebook, save_notebook
-from .interactive_example import check_binder_conf, gen_binder_rst
-from .interactive_example import check_jupyterlite_conf, gen_jupyterlite_rst
+from .interactive_example import gen_binder_rst
+from .interactive_example import gen_jupyterlite_rst
 
 
 logger = sphinx.util.logging.getLogger('sphinx-gallery')
@@ -331,7 +331,7 @@ def save_thumbnail(image_path_template, src_file, script_vars, file_conf,
         img = thumbnail_image_path
     elif not os.path.exists(thumb_file):
         # create something to replace the thumbnail
-        default_thumb_path = gallery_conf.get("default_thumb_file")
+        default_thumb_path = gallery_conf["default_thumb_file"]
         if default_thumb_path is None:
             default_thumb_path = os.path.join(
                 glr_path_static(),
@@ -510,7 +510,7 @@ def generate_dir_rst(
         subsection_index_content = None
 
     # Copy over any other files.
-    copyregex = gallery_conf.get('copyfile_regex')
+    copyregex = gallery_conf['copyfile_regex']
     if copyregex:
         listdir = [fname for fname in os.listdir(src_dir) if
                    re.match(copyregex, fname)]
@@ -911,7 +911,7 @@ def executable_script(src_file, gallery_conf):
         True if script has to be executed
     """
 
-    filename_pattern = gallery_conf.get('filename_pattern')
+    filename_pattern = gallery_conf['filename_pattern']
     execute = re.search(filename_pattern, src_file) and gallery_conf[
         'plot_gallery']
     return execute
@@ -1230,7 +1230,7 @@ def rst_blocks(script_blocks, output_blocks, file_conf, gallery_conf):
         if blabel == 'code':
 
             if not file_conf.get('line_numbers',
-                                 gallery_conf.get('line_numbers', False)):
+                                 gallery_conf['line_numbers']):
                 lineno = None
 
             code_rst = codestr2rst(bcontent, lang=gallery_conf['lang'],
@@ -1272,12 +1272,10 @@ def save_rst_example(example_rst, example_file, time_elapsed,
     example_fname = os.path.relpath(example_file, gallery_conf['src_dir'])
     ref_fname = example_fname.replace(os.path.sep, "_")
 
-    binder_conf = check_binder_conf(gallery_conf.get('binder'))
+    binder_conf = gallery_conf['binder']
     is_binder_enabled = len(binder_conf) > 0
 
-    jupyterlite_conf = check_jupyterlite_conf(
-        gallery_conf.get('jupyterlite', {}),
-        gallery_conf['app'])
+    jupyterlite_conf = gallery_conf['jupyterlite']
     is_jupyterlite_enabled = jupyterlite_conf is not None
 
     interactive_example_text = ""

sphinx_gallery/interactive_example.py

@@ -47,8 +47,8 @@ def gen_binder_url(fpath, binder_conf, gallery_conf):
         environment.
     """
     # Build the URL
-    fpath_prefix = binder_conf.get('filepath_prefix')
-    link_base = binder_conf.get('notebooks_dir')
+    fpath_prefix = binder_conf['filepath_prefix']
+    link_base = binder_conf['notebooks_dir']
 
     # We want to keep the relative path to sub-folders
     relative_link = os.path.relpath(fpath, gallery_conf['src_dir'])
@@ -69,7 +69,7 @@ def gen_binder_url(fpath, binder_conf, gallery_conf):
                            binder_conf['repo'],
                            quote(binder_conf['branch'])])
 
-    if binder_conf.get('use_jupyter_lab', False) is True:
+    if binder_conf['use_jupyter_lab'] is True:
         binder_url += f'?urlpath=lab/tree/{quote(path_link)}'
     else:
         binder_url += f'?filepath={quote(path_link)}'
@@ -154,7 +154,7 @@ def copy_binder_files(app, exception):
 
 def _copy_binder_reqs(app, binder_conf):
     """Copy Binder requirements files to a "binder" folder in the docs."""
-    path_reqs = binder_conf.get('dependencies')
+    path_reqs = binder_conf['dependencies']
     for path in path_reqs:
         if not os.path.exists(os.path.join(app.srcdir, path)):
             raise ConfigError(
@@ -197,9 +197,9 @@ def _copy_binder_notebooks(app):
     Jupyter notebook files."""
 
     gallery_conf = app.config.sphinx_gallery_conf
-    gallery_dirs = gallery_conf.get('gallery_dirs')
-    binder_conf = gallery_conf.get('binder')
-    notebooks_dir = os.path.join(app.outdir, binder_conf.get('notebooks_dir'))
+    gallery_dirs = gallery_conf['gallery_dirs']
+    binder_conf = gallery_conf['binder']
+    notebooks_dir = os.path.join(app.outdir, binder_conf['notebooks_dir'])
     shutil.rmtree(notebooks_dir, ignore_errors=True)
     os.makedirs(notebooks_dir)
 
@@ -218,7 +218,7 @@ def _copy_binder_notebooks(app):
 def check_binder_conf(binder_conf):
     """Check to make sure that the Binder configuration is correct."""
     # Grab the configuration and return None if it's not configured
-    binder_conf = {} if binder_conf is None else binder_conf
+    binder_conf = {} if binder_conf is None else binder_conf.copy()
     if not isinstance(binder_conf, dict):
         raise ConfigError('`binder_conf` must be a dictionary or None.')
     if len(binder_conf) == 0:
@@ -258,8 +258,9 @@ def check_binder_conf(binder_conf):
         raise ConfigError("`dependencies` value should be a list of strings. "
                           "Got type {}.".format(type(path_reqs)))
 
-    binder_conf['notebooks_dir'] = binder_conf.get('notebooks_dir',
-                                                   'notebooks')
+    binder_conf.setdefault('filepath_prefix')
+    binder_conf.setdefault('notebooks_dir', 'notebooks')
+    binder_conf.setdefault('use_jupyter_lab', False)
     path_reqs_filenames = [os.path.basename(ii) for ii in path_reqs]
     if not any(ii in path_reqs_filenames for ii in required_reqs_files):
         raise ConfigError(
@@ -273,7 +274,7 @@ def check_binder_conf(binder_conf):
 def pre_configure_jupyterlite_sphinx(app, config):
     is_jupyterlite_enabled = (
         'jupyterlite_sphinx' in app.extensions
-        and config.sphinx_gallery_conf.get('jupyterlite', {}) is not None
+        and config.sphinx_gallery_conf['jupyterlite'] is not None
     )
     if not is_jupyterlite_enabled:
         return
@@ -286,7 +287,7 @@ def pre_configure_jupyterlite_sphinx(app, config):
 
 def post_configure_jupyterlite_sphinx(app, config):
     config.sphinx_gallery_conf['jupyterlite'] = check_jupyterlite_conf(
-        config.sphinx_gallery_conf.get('jupyterlite', {}), app)
+        config.sphinx_gallery_conf['jupyterlite'], app)
 
     if config.sphinx_gallery_conf['jupyterlite'] is None:
         return
@@ -315,7 +316,7 @@ def create_jupyterlite_contents(app, exception):
         return
 
     logger.info('copying Jupyterlite contents ...', color='white')
-    gallery_dirs = gallery_conf.get('gallery_dirs')
+    gallery_dirs = gallery_conf['gallery_dirs']
     contents_dir = gallery_conf['jupyterlite']['jupyterlite_contents']
 
     shutil.rmtree(contents_dir, ignore_errors=True)
@@ -383,7 +384,7 @@ def gen_jupyterlite_rst(fpath, gallery_conf):
     # Make sure we have the right slashes (in case we're on Windows)
     lite_root_url = lite_root_url.replace(os.path.sep, '/')
 
-    if gallery_conf["jupyterlite"].get("use_jupyter_lab", True):
+    if gallery_conf["jupyterlite"]["use_jupyter_lab"]:
         lite_root_url += "/lab"
     else:
         lite_root_url += "/retro/notebooks"

sphinx_gallery/scrapers.py

@@ -117,7 +117,7 @@ def matplotlib_scraper(block, block_vars, gallery_conf, **kwargs):
     image_path_iterator = block_vars['image_path_iterator']
     image_rsts = []
     # Check for srcset hidpi images
-    srcset = gallery_conf.get('image_srcset', [])
+    srcset = gallery_conf['image_srcset']
     srcset_mult_facs = [1]  # one is always supplied...
     for st in srcset:
         if (len(st) > 0) and (st[-1] == 'x'):
@@ -133,7 +133,7 @@ def matplotlib_scraper(block, block_vars, gallery_conf, **kwargs):
 
     # Check for animations
     anims = list()
-    if gallery_conf.get('matplotlib_animations', False):
+    if gallery_conf['matplotlib_animations']:
         for ani in block_vars['example_globals'].values():
             if isinstance(ani, Animation):
                 anims.append(ani)

sphinx_gallery/tests/conftest.py

@@ -29,8 +29,10 @@ def gallery_conf(tmpdir):
     """Set up a test sphinx-gallery configuration."""
     app = Mock(spec=Sphinx, config=dict(source_suffix={'.rst': None}),
                extensions=[])
-    gallery_conf = gen_gallery._complete_gallery_conf(
-        {}, str(tmpdir), True, False, app=app)
+    gallery_conf = gen_gallery._complete_gallery_conf_config_inited(
+        {},  app=app)
+    gen_gallery._complete_gallery_conf_builder_inited(
+        gallery_conf, str(tmpdir))
     gallery_conf.update(examples_dir=str(tmpdir), gallery_dir=str(tmpdir))
     return gallery_conf