feat(python_samples): Allow for additional README generation in speci…

…fied path(s) (#724)

* feat(python_samples): Custom path for generation

* Indentation

* Indentation & Typo

* feat(python_samples): Custom path for generation

* Indentation

* Indentation & Typo

* Updates based on Review

* Comment update

* Added testing for common.py_samples()

* Repair filenames & minor fixes

* Updates to exclude and testing method

* Test alterations

* Linting

* Linting

* Add exception handling

* Resolve test failure

* Linting

* Linting

* WIP: Multiple paths override, saving work

* Revert "WIP: Multiple paths override, saving work"

This reverts commit 4fe194e.

* WIP

* WIP

* Docs, cleanup, linting

* Revert test_git.py

* Fix requested changes

* Linting fix

* Blacken/Lint

* More linting

Co-authored-by: kolea2 <45548808+kolea2@users.noreply.github.com>

synthtool/gcp/common.py

@@ -17,9 +17,9 @@
 import re
 import shutil
 import yaml
+from copy import deepcopy
 from pathlib import Path
 from typing import Dict, List, Optional
-
 import jinja2
 
 from synthtool import _tracked_paths
@@ -62,18 +62,22 @@ def _generic_library(self, directory: str, **kwargs) -> Path:
 
         return result
 
-    def py_samples(self, **kwargs) -> Path:
+    def py_samples(self, **kwargs) -> List[Path]:
         """
-        Determines whether generation is being done in a client library or in a samples
-        folder so it can either generate in the current directory or the client lib's
-        'samples' folder. A custom path for where to generate may also be specified.
-        Renders README.md according to .repo-metadata.json
+        Handles generation of README.md templates for Python samples
+        - Determines whether generation is being done in a client library or in a samples
+        folder automatically
+        - Otherwise accepts manually set sample_project_dir through kwargs metadata
+        - Delegates generation of additional sample documents alternate/overridden folders
+        through py_samples_override()
         """
         # kwargs["metadata"] is required to load values from .repo-metadata.json
         if "metadata" not in kwargs:
             kwargs["metadata"] = {}
+
         # load common repo meta information (metadata that's not language specific).
         self._load_generic_metadata(kwargs["metadata"])
+
         # temporary exclusion prior to old templates being migrated out
         self.excludes.extend(
             [
@@ -86,6 +90,7 @@ def py_samples(self, **kwargs) -> Path:
             ]
         )
 
+        # determine if in client lib and set custom root sample dir if specified, else None
         in_client_library = Path("samples").exists()
         sample_project_dir = kwargs["metadata"]["repo"].get("sample_project_dir")
 
@@ -97,15 +102,85 @@ def py_samples(self, **kwargs) -> Path:
         elif not Path(sample_project_dir).exists():
             raise Exception(f"'{sample_project_dir}' does not exist")
 
+        override_paths_to_samples: Dict[
+            str, List[str]
+        ] = {}  # Dict of format { override_path : sample(s) }
+        samples_dict = deepcopy(kwargs["metadata"]["repo"].get("samples"))
+        default_samples_dict = []  # Dict which will generate in sample_project_dir
+
+        # Iterate through samples to store override_paths_to_samples for all existing
+        # override paths
+        for sample_idx, sample in enumerate(samples_dict):
+            override_path = samples_dict[sample_idx].get("override_path")
+            if (
+                override_path is not None
+            ):  # sample should be placed in an override README
+                cur_override_sample = override_paths_to_samples.get(override_path)
+                # Base case: No samples are yet planned to gen in this override dir
+                if cur_override_sample is None:
+                    override_paths_to_samples[override_path] = [sample]
+                # Else: Sample docs will be generated in README merged with other
+                # sample doc(s) already planned to generate in this dir
+                else:
+                    cur_override_sample.append(sample)
+                    override_paths_to_samples[override_path] = cur_override_sample
+            # If override path none, will be generated in the default
+            # folder: sample_project_dir
+            else:
+                default_samples_dict.append(sample)
+
+        result = (
+            []
+        )  # List of paths to tempdirs which will be copied into sample folders
+        overridden_samples_kwargs = deepcopy(
+            kwargs
+        )  # deep copy is req. here to avoid kwargs being affected
+        for override_path in override_paths_to_samples:
+            # Generate override sample docs
+            result.append(
+                self.py_samples_override(
+                    root=sample_project_dir,
+                    override_path=override_path,
+                    override_samples=override_paths_to_samples[override_path],
+                    **overridden_samples_kwargs,
+                )
+            )
+        kwargs["metadata"]["repo"]["samples"] = default_samples_dict
+
         logger.debug(
             f"Generating templates for samples directory '{sample_project_dir}'"
         )
-        py_samples_templates = Path(self._template_root) / "python_samples"
-        t = templates.TemplateGroup(py_samples_templates, self.excludes)
-        result = t.render(subdir=sample_project_dir, **kwargs)
-        _tracked_paths.add(result)
+        kwargs["subdir"] = sample_project_dir
+        # Generate default sample docs
+        result.append(self._generic_library("python_samples", **kwargs))
+
+        for path in result:
+            # .add() records the root of the paths and needs to be applied to each
+            _tracked_paths.add(path)
+
         return result
 
+    def py_samples_override(
+        self, root, override_path, override_samples, **overridden_samples_kwargs
+    ) -> Path:
+        """
+        Handles additional generation of READMEs where "override_path"s
+        are set in one or more samples' metadata
+        """
+        overridden_samples_kwargs["metadata"]["repo"][
+            "sample_project_dir"
+        ] = override_path
+        # Set samples metadata to ONLY samples intended to generate
+        # under this directory (override_path)
+        overridden_samples_kwargs["metadata"]["repo"]["samples"] = override_samples
+        if root != ".":
+            override_path = Path(root) / override_path
+
+        logger.debug(f"Generating templates for override path '{override_path}'")
+
+        overridden_samples_kwargs["subdir"] = override_path
+        return self._generic_library("python_samples", **overridden_samples_kwargs)
+
     def py_library(self, **kwargs) -> Path:
         # kwargs["metadata"] is required to load values from .repo-metadata.json
         if "metadata" not in kwargs:

tests/generationmock/multiple_override_path/.repo-metadata.json

@@ -0,0 +1,33 @@
+{
+    "name": "bigtable",
+    "name_pretty": "Cloud Bigtable",
+    "product_documentation": "https://cloud.google.com/bigtable",
+    "client_documentation": "https://googleapis.dev/python/bigtable/latest",
+    "issue_tracker": "https://issuetracker.google.com/savedsearches/559777",
+    "release_level": "ga",
+    "language": "python",
+    "repo": "googleapis/python-bigtable",
+    "distribution_name": "google-cloud-bigtable",
+    "api_id": "bigtable.googleapis.com",
+    "requires_billing": true,
+    "client_library": true,
+    "samples": [
+              {"name": "Quickstart",
+              "description": "Example sample for product",
+              "file": "quickstart.py",
+              "custom_content": "This is custom text for the sample",
+              "runnable": "True",
+              "override_path": "override"},
+              {"name": "Hello World",
+              "description": "Example beginner application",
+              "file": "main.py",
+              "override_path": "another_override"},
+              {"name": "Hello Synthtool",
+              "description": "Example application",
+              "file": "main.py",
+              "override_path": "another_override"},
+              {"name": "Last Example",
+              "description": "Last application",
+              "file": "example.py"}
+    ]
+  }

tests/generationmock/override_path/.repo-metadata.json

@@ -0,0 +1,25 @@
+{
+    "name": "bigtable",
+    "name_pretty": "Cloud Bigtable",
+    "product_documentation": "https://cloud.google.com/bigtable",
+    "client_documentation": "https://googleapis.dev/python/bigtable/latest",
+    "issue_tracker": "https://issuetracker.google.com/savedsearches/559777",
+    "release_level": "ga",
+    "language": "python",
+    "repo": "googleapis/python-bigtable",
+    "distribution_name": "google-cloud-bigtable",
+    "api_id": "bigtable.googleapis.com",
+    "requires_billing": true,
+    "client_library": true,
+    "samples": [
+              {"name": "Quickstart",
+              "description": "Example sample for product",
+              "file": "quickstart.py",
+              "custom_content": "This is custom text for the sample",
+              "runnable": "True",
+              "override_path": "override"},
+              {"name": "Hello World",
+              "description": "Example beginner application",
+              "file": "main.py"}
+    ]
+  }

tests/test_common.py

@@ -109,3 +109,104 @@ def test_py_samples_samples_folder():
             assert os.path.isfile(workdir / "README.md")
         finally:
             os.chdir(cwd)
+
+
+def test_py_samples_override():
+    path_to_gen = MOCK / "override_path"
+    with tempfile.TemporaryDirectory() as tempdir:
+        workdir = shutil.copytree(path_to_gen, Path(tempdir) / "override_path")
+        cwd = os.getcwd()
+        os.chdir(workdir)
+
+        try:
+            sample_files = common.py_samples(
+                unit_cov_level=97, cov_level=99, samples=True
+            )
+            for path in sample_files:
+                s.move(path, excludes=["noxfile.py"])
+            assert os.path.isfile(workdir / "README.md")
+            assert os.path.isfile(workdir / "override" / "README.md")
+        finally:
+            os.chdir(cwd)
+
+
+def test_py_samples_override_content():
+    path_to_gen = MOCK / "override_path"
+    with tempfile.TemporaryDirectory() as tempdir:
+        workdir = shutil.copytree(path_to_gen, Path(tempdir) / "override_path")
+        cwd = os.getcwd()
+        os.chdir(workdir)
+
+        try:
+            sample_files = common.py_samples(
+                unit_cov_level=97, cov_level=99, samples=True
+            )
+            for path in sample_files:
+                s.move(path, excludes=["noxfile.py"])
+            os.chdir(workdir)
+            with open("README.md") as f:
+                result = f.read()
+                assert "Hello World" in result
+                assert "Quickstart" not in result
+            os.chdir(workdir / "override")
+            with open("README.md") as f:
+                result = f.read()
+                assert "Hello World" not in result
+                assert "Quickstart" in result
+        finally:
+            os.chdir(cwd)
+
+
+def test_py_samples_multiple_override():
+    path_to_gen = MOCK / "multiple_override_path"
+    with tempfile.TemporaryDirectory() as tempdir:
+        workdir = shutil.copytree(path_to_gen, Path(tempdir) / "multiple_override_path")
+        cwd = os.getcwd()
+        os.chdir(workdir)
+
+        try:
+            sample_files = common.py_samples(
+                unit_cov_level=97, cov_level=99, samples=True
+            )
+            for path in sample_files:
+                s.move(path, excludes=["noxfile.py"])
+            assert os.path.isfile(workdir / "README.md")
+            assert os.path.isfile(workdir / "override" / "README.md")
+            assert os.path.isfile(workdir / "another_override" / "README.md")
+            os.chdir(workdir / "another_override")
+            with open("README.md") as f:
+                result = f.read()
+                assert "Hello World" in result
+                assert "Hello Synthtool" in result
+        finally:
+            os.chdir(cwd)
+
+
+def test_py_samples_multiple_override_content():
+    path_to_gen = MOCK / "multiple_override_path"
+    with tempfile.TemporaryDirectory() as tempdir:
+        workdir = shutil.copytree(path_to_gen, Path(tempdir) / "multiple_override_path")
+        cwd = os.getcwd()
+        os.chdir(workdir)
+
+        try:
+            sample_files = common.py_samples(
+                unit_cov_level=97, cov_level=99, samples=True
+            )
+            for path in sample_files:
+                s.move(path, excludes=["noxfile.py"])
+            os.chdir(workdir / "override")
+            with open("README.md") as f:
+                result = f.read()
+                assert "Quickstart" in result
+            os.chdir(workdir / "another_override")
+            with open("README.md") as f:
+                result = f.read()
+                assert "Hello World" in result
+                assert "Hello Synthtool" in result
+            os.chdir(workdir)
+            with open("README.md") as f:
+                result = f.read()
+                assert "Last Example" in result
+        finally:
+            os.chdir(cwd)