Merge branch 'latest' into ea/add_toc_and_device_check

.ci/check_links.py

@@ -84,21 +84,13 @@ def complain(message):
             try:
                 get = requests.get(url, timeout=10)
                 if get.status_code != 200:
-                    if get.status_code in [500, 429, 443] and any(
-                        [known_url in url for known_url in EXCEPTIONS_URLs]
-                    ):
-                        print(
-                            f"SKIP - {md_path}: URL can not be reached {url!r}, status code {get.status_code}"
-                        )
+                    if get.status_code in [500, 429, 443] and any([known_url in url for known_url in EXCEPTIONS_URLs]):
+                        print(f"SKIP - {md_path}: URL can not be reached {url!r}, status code {get.status_code}")
                         continue
-                    complain(
-                        f"{md_path}: URL can not be reached {url!r}, status code {get.status_code}"
-                    )
+                    complain(f"{md_path}: URL can not be reached {url!r}, status code {get.status_code}")
             except Exception as err:
                 if any([known_url in url for known_url in EXCEPTIONS_URLs]):
-                    print(
-                        f"SKIP - {md_path}: URL can not be reached {url!r}, error {err}"
-                    )
+                    print(f"SKIP - {md_path}: URL can not be reached {url!r}, error {err}")
                 else:
                     complain(f"{md_path}: URL can not be reached {url!r}, error {err}")
 

.ci/check_notebooks.py

@@ -3,8 +3,10 @@
 from table_of_content import find_tc_in_cell
 from patch_notebooks import DEVICE_WIDGET
 from pathlib import Path
+
 NOTEBOOKS_ROOT = Path(__file__).resolve().parents[1]
 
+
 def main():
     all_passed = True
     no_tocs = []
@@ -21,12 +23,12 @@ def complain(message):
             toc_found = False
             device_found = False
             for cell in notebook_json["cells"]:
-                
+
                 if not toc_found and cell["cell_type"] == "markdown":
                     tc_cell, tc_line = find_tc_in_cell(cell)
                     if tc_line is not None:
                         toc_found = True
-                
+
                 if not device_found and cell["cell_type"] == "code":
                     device_found = DEVICE_WIDGET in cell["source"]
 
@@ -38,7 +40,6 @@ def complain(message):
             if not device_found:
                 no_device.append(nb_path.relative_to(NOTEBOOKS_ROOT))
                 complain(f"FAILEd: {nb_path.relative_to(NOTEBOOKS_ROOT)}: device widget is not found")
-
 
     if not all_passed:
         print("SUMMARY:")
@@ -48,9 +49,9 @@ def complain(message):
         print("==================================")
         print("NO DEVICE SELECTION:")
         print("\n".join(no_device))
-    
+
     sys.exit(0 if all_passed else 1)
 
+
 if __name__ == "__main__":
     main()
-

.ci/convert_notebooks.py

@@ -23,12 +23,8 @@ def arguments():
     parser = argparse.ArgumentParser()
     parser.add_argument("--exclude_execution_file")
     parser.add_argument("--exclude_conversion_file")
-    parser.add_argument(
-        "--timeout", type=float, default=7200, help="timeout for notebook execution"
-    )
-    parser.add_argument(
-        "--rst_dir", type=Path, help="rst files output directory", default=Path("rst")
-    )
+    parser.add_argument("--timeout", type=float, default=7200, help="timeout for notebook execution")
+    parser.add_argument("--rst_dir", type=Path, help="rst files output directory", default=Path("rst"))
 
     return parser.parse_args()
 
@@ -57,9 +53,7 @@ def main():
         if str(notebook_path) in ignore_conversion_list:
             continue
         disable_gradio_debug(notebook_path)
-        notebook_executed = notebook_path.parent / notebook_path.name.replace(
-            ".ipynb", "-with-output.ipynb"
-        )
+        notebook_executed = notebook_path.parent / notebook_path.name.replace(".ipynb", "-with-output.ipynb")
         start = time.perf_counter()
         print(f"Convert {notebook_path}")
         if str(notebook_path) not in ignore_execution_list:

.ci/patch_notebooks.py

@@ -52,24 +52,17 @@ def patch_notebooks(notebooks_dir, test_device=""):
     """
 
     nb_convert_config = Config()
-    nb_convert_config.NotebookExporter.preprocessors = [
-        "nbconvert.preprocessors.ClearOutputPreprocessor"
-    ]
+    nb_convert_config.NotebookExporter.preprocessors = ["nbconvert.preprocessors.ClearOutputPreprocessor"]
     output_remover = nbconvert.NotebookExporter(nb_convert_config)
     for notebookfile in Path(notebooks_dir).glob("**/*.ipynb"):
-        if (
-            not str(notebookfile.name).startswith("test_")
-            and notebookfile.name not in EXCLUDED_NOTEBOOKS
-        ):
+        if not str(notebookfile.name).startswith("test_") and notebookfile.name not in EXCLUDED_NOTEBOOKS:
             nb = nbformat.read(notebookfile, as_version=nbformat.NO_CONVERT)
             found = False
             device_found = False
             for cell in nb["cells"]:
                 if test_device and DEVICE_WIDGET in cell["source"]:
                     device_found = True
-                    cell["source"] = re.sub(
-                        r"value=.*,", f"value='{test_device.upper()}',", cell["source"]
-                    )
+                    cell["source"] = re.sub(r"value=.*,", f"value='{test_device.upper()}',", cell["source"])
                     cell["source"] = re.sub(
                         r"options=",
                         f"options=['{test_device.upper()}'] + ",
@@ -81,26 +74,18 @@ def patch_notebooks(notebooks_dir, test_device=""):
                     found = True
                     for source_value, target_value in replace_dict.items():
                         if source_value not in cell["source"]:
-                            raise ValueError(
-                                f"Processing {notebookfile} failed: {source_value} does not exist in cell"
-                            )
-                        cell["source"] = cell["source"].replace(
-                            source_value, target_value
-                        )
+                            raise ValueError(f"Processing {notebookfile} failed: {source_value} does not exist in cell")
+                        cell["source"] = cell["source"].replace(source_value, target_value)
                         cell["source"] = "# Modified for testing\n" + cell["source"]
-                        print(
-                            f"Processed {notebookfile}: {source_value} -> {target_value}"
-                        )
+                        print(f"Processed {notebookfile}: {source_value} -> {target_value}")
             if test_device and not device_found:
                 print(f"No device replacement found for {notebookfile}")
             if not found:
                 print(f"No replacements found for {notebookfile}")
             disable_gradio_debug(nb, notebookfile)
             disable_skip_ext(nb, notebookfile)
             nb_without_out, _ = output_remover.from_notebook_node(nb)
-            with notebookfile.with_name(f"test_{notebookfile.name}").open(
-                "w", encoding="utf-8"
-            ) as out_file:
+            with notebookfile.with_name(f"test_{notebookfile.name}").open("w", encoding="utf-8") as out_file:
                 out_file.write(nb_without_out)
 
 

.ci/spellcheck/ipynb_filter.py

@@ -25,9 +25,7 @@ def setup(self):
 
     def filter(self, source_file, encoding):  # noqa A001
         """Open and filter the file from disk."""
-        nb: nbformat.NotebookNode = nbformat.read(
-            source_file, as_version=nbformat.NO_CONVERT
-        )
+        nb: nbformat.NotebookNode = nbformat.read(source_file, as_version=nbformat.NO_CONVERT)
 
         return [filters.SourceText(self._filter(nb), source_file, encoding, "ipynb")]
 
@@ -42,11 +40,7 @@ def _filter(self, nb):
 
     def sfilter(self, source):
         """Execute filter."""
-        return [
-            filters.SourceText(
-                self._filter(source.text), source.context, source.encoding, "ipynb"
-            )
-        ]
+        return [filters.SourceText(self._filter(source.text), source.context, source.encoding, "ipynb")]
 
 
 def get_plugin():

.ci/spellcheck/run_spellcheck.py

@@ -18,9 +18,7 @@
     env=dict(os.environ, PYTHONPATH=PYTHONPATH),
 )
 
-result_output = (
-    result.stdout.strip("\n") if result.stdout else result.stderr.strip("\n")
-)
+result_output = result.stdout.strip("\n") if result.stdout else result.stderr.strip("\n")
 
 print(result_output, file=sys.stderr if result.returncode else sys.stdout, flush=True)
 

.ci/table_of_content.py

@@ -36,9 +36,7 @@ def create_link_for_tc(title):
 def remove_old_tc(cell, idx):
     if cell is not None:
         for line in cell["source"][idx:]:
-            if re.match(r"\s*-\s*\[.*\]\(#.*\).*", line) or re.match(
-                TABLE_OF_CONTENT, line
-            ):
+            if re.match(r"\s*-\s*\[.*\]\(#.*\).*", line) or re.match(TABLE_OF_CONTENT, line):
                 cell["source"].remove(line)
     return cell
 
@@ -56,9 +54,7 @@ def get_tc_line(title, title_for_tc, link, tc_list, titles_list):
     elif indents_num - tc_list[-1].index("-") > 4:
         # when previous list item have n indents and current have n+4+1 it broke the alignment
         indents_num = tc_list[-1].index("-") + 4
-    elif indents_num != tc_list[-1].index("-") and title.index(" ") == titles_list[
-        -1
-    ].index(" "):
+    elif indents_num != tc_list[-1].index("-") and title.index(" ") == titles_list[-1].index(" "):
         # when we have several titles with same wrong alignments
         indents_num = tc_list[-1].index("-")
 
@@ -101,9 +97,7 @@ def generate_table_of_content(notebook_path: pathlib.Path):
     if not notebook_json["cells"]:
         return
 
-    table_of_content_cell, table_of_content_cell_idx = find_tc_in_cell(
-        notebook_json["cells"][0]
-    )
+    table_of_content_cell, table_of_content_cell_idx = find_tc_in_cell(notebook_json["cells"][0])
 
     all_titles = []
     for cell in filter(is_markdown, notebook_json["cells"][1:]):
@@ -124,9 +118,7 @@ def generate_table_of_content(notebook_path: pathlib.Path):
             title = title.strip()
             title_for_tc = create_title_for_tc(title)
             link_for_tc = create_link_for_tc(title_for_tc)
-            new_line = get_tc_line(
-                title, title_for_tc, link_for_tc, table_of_content, all_titles
-            )
+            new_line = get_tc_line(title, title_for_tc, link_for_tc, table_of_content, all_titles)
 
             if table_of_content.count(new_line) > 1:
                 print(
@@ -141,9 +133,7 @@ def generate_table_of_content(notebook_path: pathlib.Path):
     table_of_content = ["\n", "#### Table of contents:\n\n"] + table_of_content + ["\n"]
 
     if table_of_content_cell is not None:
-        table_of_content_cell = remove_old_tc(
-            table_of_content_cell, table_of_content_cell_idx
-        )
+        table_of_content_cell = remove_old_tc(table_of_content_cell, table_of_content_cell_idx)
 
     if table_of_content_cell is not None:
         table_of_content_cell["source"].extend(table_of_content)

.ci/test_notebooks.py

@@ -45,9 +45,7 @@ def test_readme():
             # item is a notebook directory
             notebook_dir = item.relative_to(Path("notebooks"))
             if str(notebook_dir)[0].isdigit():
-                assert "README.md" in [
-                    filename.name for filename in item.iterdir()
-                ], f"README not found in {item}"
+                assert "README.md" in [filename.name for filename in item.iterdir()], f"README not found in {item}"
 
 
 def test_requirements_docker():
@@ -60,9 +58,7 @@ def test_requirements_docker():
         docker_requirements = set(list(pipfile_contents["packages"].keys()))
 
     pip_requirements = get_parsed_requirements("requirements.txt")
-    assert pip_requirements.issubset(
-        docker_requirements
-    ), f"Docker Pipfile misses: {pip_requirements.difference(docker_requirements)}"
+    assert pip_requirements.issubset(docker_requirements), f"Docker Pipfile misses: {pip_requirements.difference(docker_requirements)}"
 
 
 def test_requirements_binder():
@@ -72,9 +68,7 @@ def test_requirements_binder():
     """
     pip_requirements = get_parsed_requirements("requirements.txt")
     binder_requirements = get_parsed_requirements(".binder/requirements.txt")
-    assert pip_requirements.issubset(
-        binder_requirements
-    ), f"Binder requirements misses: {pip_requirements.difference(binder_requirements)}"
+    assert pip_requirements.issubset(binder_requirements), f"Binder requirements misses: {pip_requirements.difference(binder_requirements)}"
 
 
 @pytest.mark.skip(reason="URL existence is tested in docker_treon")

.ci/validate_notebooks.py

@@ -58,9 +58,7 @@ def prepare_test_plan(test_list, ignore_list, nb_dir=None):
         for ig_nb in ignore_list:
             if ig_nb.endswith(".txt"):
                 with open(ig_nb, "r") as f:
-                    ignored_notebooks.extend(
-                        list(map(lambda x: x.strip(), f.readlines()))
-                    )
+                    ignored_notebooks.extend(list(map(lambda x: x.strip(), f.readlines())))
             else:
                 ignored_notebooks.append(ig_nb)
         print(f"ignored notebooks: {ignored_notebooks}")
@@ -76,9 +74,7 @@ def prepare_test_plan(test_list, ignore_list, nb_dir=None):
                     break
                 if changed_path.suffix == ".md":
                     continue
-                notebook_subdir = find_notebook_dir(
-                    changed_path.resolve(), orig_nb_dir.resolve()
-                )
+                notebook_subdir = find_notebook_dir(changed_path.resolve(), orig_nb_dir.resolve())
                 if notebook_subdir is None:
                     continue
                 testing_notebooks.append(notebook_subdir)
@@ -189,9 +185,7 @@ def main():
     if args.keep_artifacts:
         keep_artifacts = True
 
-    test_plan = prepare_test_plan(
-        args.test_list, args.ignore_list, notebooks_moving_dir
-    )
+    test_plan = prepare_test_plan(args.test_list, args.ignore_list, notebooks_moving_dir)
     for notebook, report in test_plan.items():
         if report["status"] == "SKIPPED":
             continue
@@ -203,21 +197,15 @@ def main():
             if status:
                 report["status"] = "TIMEOUT" if status == -42 else "FAILED"
             else:
-                report["status"] = (
-                    "SUCCESS"
-                    if not report["status"] in ["TIMEOUT", "FAILED"]
-                    else report["status"]
-                )
+                report["status"] = "SUCCESS" if not report["status"] in ["TIMEOUT", "FAILED"] else report["status"]
             if status:
                 if status == -42:
                     timeout_notebooks.append(str(subnotebook))
                 else:
                     failed_notebooks.append(str(subnotebook))
             if args.early_stop:
                 break
-    exit_status = finalize_status(
-        failed_notebooks, timeout_notebooks, test_plan, reports_dir, root
-    )
+    exit_status = finalize_status(failed_notebooks, timeout_notebooks, test_plan, reports_dir, root)
     return exit_status
 
 

.github/workflows/codecheck.yml

@@ -82,7 +82,7 @@ jobs:
           nbqa flake8 --ignore=E124,E203,E231,E266,E402,E501,E703,F821,W503,W291,W293 --nbqa-exclude="(tensorflow-training-openvino)|(pytorch-quantization-aware-training)" notebooks
       - name: Black
         run: |
-          black --check .
+          black --check -l 160 .
       - name: Test READMEs and requirements
         run: |
           python -m pytest .ci/test_notebooks.py