Merge branch 'dev'

Author: boonhapus

CONTRIBUTING.md

@@ -79,6 +79,23 @@ requires some extra dependencies.
 uv pip install -e ".[docs]"
 ```
 
+Set your environment variables so that the generated documentation (`hooks/cli_reference_generator.py`) can be built
+against a valid ThoughtSpot cluster.
+
+`Windows`
+```powershell
+$env:CS_TOOLS_THOUGHTSPOT__URL = "https://<YOUR-THOUGHTSPOT-CLUSTER>.thoughtspot.cloud"
+$env:CS_TOOLS_THOUGHTSPOT__USERNAME = "<YOUR-THOUGHTSPOT-USERNAME>"
+$env:CS_TOOLS_THOUGHTSPOT__PASSWORD = "<YOUR-THOUGHTSPOT-PASSWORD>"
+```
+
+`POSIX (Mac, Linux)`
+```bash
+CS_TOOLS_THOUGHTSPOT__URL = "https://<YOUR-THOUGHTSPOT-CLUSTER>.thoughtspot.cloud"
+CS_TOOLS_THOUGHTSPOT__USERNAME = "<YOUR-THOUGHTSPOT-USERNAME>"
+CS_TOOLS_THOUGHTSPOT__PASSWORD = "<YOUR-THOUGHTSPOT-PASSWORD>"
+```
+
 Note that the [__CS Tools__ website](https://thoughtspot.github.io/cs_tools/) is only updated when a new version is
 released so your contribution might not show up for a while.
 

cs_tools/cli/_logging.py

@@ -53,6 +53,7 @@ def __init__(self):
             "secret_key": (r'secret_key[\'"]\s*:\s*[\'"][^\'\"]+[\'"]', r'secret_key": "****"'),
             "token": (r'token[\'"]\s*:\s*[\'"][^\'\"]+[\'"]', r'token": "****"'),
             "authorization": (r"authorization\s*:\s*bearer\s+\S+", r"authorization: bearer ****"),
+            "jsessionid": (r"JSESSIONID=[^\'\"]+;", r"JESSIONID=****"),
             # ADD MORE PATTERNS AS NEEDED
         }
 

cs_tools/cli/commands/log.py

@@ -19,7 +19,7 @@ def report(
         ...,
         metavar="DIRECTORY",
         help="Where to export the logs.",
-        click_type=custom_types.Directory(exists=True, make=True),
+        click_type=custom_types.Directory(exists=False, make=True),
     ),
     latest: int = typer.Option(1, help="Number of most recent logfiles to export.", min=1),
 ) -> _types.ExitCode:

cs_tools/cli/commands/main.py

@@ -4,7 +4,7 @@
 import logging
 import random
 
-from cs_tools import __project__, __version__, _compat, datastructures, errors
+from cs_tools import __project__, __version__, _compat, _types, datastructures, errors
 from cs_tools.cli._logging import _setup_logging
 from cs_tools.cli.ux import RICH_CONSOLE, AsyncTyper
 from cs_tools.settings import _meta_config as meta
@@ -52,10 +52,8 @@ def main(version: bool = typer.Option(False, "--version", help="Show the version
         raise typer.Exit(0)
 
 
-def run() -> int:
-    """
-    Entrypoint into cs_tools.
-    """
+def run() -> _types.ExitCode:
+    """Entrypoint into cs_tools."""
     from cs_tools.cli import _monkey  # noqa: F401
     from cs_tools.cli.commands import (
         config as config_command,
@@ -85,7 +83,7 @@ def run() -> int:
 
     except click.ClickException as e:
         return_code = 1
-        log.error(e)
+        log.error(f"{e.format_message()}")
         log.debug("More info..", exc_info=True)
 
     except errors.CSToolsError as e:

cs_tools/cli/commands/self.py

@@ -34,7 +34,7 @@ def info(
     directory: pathlib.Path = typer.Option(
         None,
         help="Where to export the info to share with the CS Tools team.",
-        click_type=custom_types.Directory(exists=True),
+        click_type=custom_types.Directory(exists=False, make=True),
     ),
     anonymous: bool = typer.Option(False, "--anonymous", help="remove personal references from the output"),
 ) -> _types.ExitCode:
@@ -102,11 +102,13 @@ def sync() -> _types.ExitCode:
 @app.command(name="update")
 def update(
     beta: custom_types.Version = typer.Option(None, "--beta", help="The specific beta version to fetch from Github."),
-    offline: custom_types.Directory = typer.Option(None, help="Install cs_tools from a local directory."),
+    offline: pathlib.Path = typer.Option(
+        None,
+        help="Install cs_tools from a local directory.",
+        click_type=custom_types.Directory(),
+    ),
 ) -> _types.ExitCode:
     """Upgrade CS Tools."""
-    assert isinstance(offline, pathlib.Path), "offline directory must be a pathlib.Path"
-
     if offline is not None:
         cs_tools_venv.offline_index = offline
         where = offline.as_posix()
@@ -122,7 +124,11 @@ def update(
 @app.command(name="export", hidden=True)
 @app.command(name="download")
 def _make_offline_distributable(
-    directory: custom_types.Directory = typer.Option(help="Location to export the python distributable to."),
+    directory: pathlib.Path = typer.Option(
+        ...,
+        help="Location to export the python distributable to.",
+        click_type=custom_types.Directory(exists=False, make=True),
+    ),
     platform: str = typer.Option(help="A tag describing the target environment architecture, see --help for details."),
     python_version: custom_types.Version = typer.Option(
         metavar="X.Y", help="The major and minor version of the target Python environment, see --help for details"
@@ -139,6 +145,9 @@ def _make_offline_distributable(
 
     Q. How can I find my python version?
     >>> [fg-secondary]python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}')"[/]
+
+    Q. Do I need anything other than python?
+    A. You also likely need a rust compiler, which can be installed via Rust ( https://www.rust-lang.org/tools/install ).
     """
     assert isinstance(directory, pathlib.Path), "directory must be a pathlib.Path"
 

cs_tools/cli/custom_types.py

@@ -9,13 +9,12 @@
 
 from awesomeversion import AwesomeVersion, AwesomeVersionStrategy, AwesomeVersionStrategyException
 import click
-import pydantic
 import toml
 
 from cs_tools import datastructures, utils
 from cs_tools.sync import base
 
-log = logging.getLogger(__name__)
+_LOG = logging.getLogger(__name__)
 
 
 class CustomType(click.ParamType):
@@ -113,7 +112,7 @@ def convert(self, value: Any, param: Optional[click.Parameter], ctx: Optional[cl
 class Directory(CustomType):
     """Convert STR to DIRECTORY PATH."""
 
-    def __init__(self, exists: bool = False, make: bool = False):
+    def __init__(self, exists: bool = True, make: bool = False):
         self.exists = exists
         self.make = make
 
@@ -124,14 +123,15 @@ def get_metavar(self, param: click.Parameter) -> str:  # noqa: ARG002
     def convert(self, value: Any, param: Optional[click.Parameter], ctx: Optional[click.Context]) -> pathlib.Path:
         """Coerce string into a pathlib.Path.is_dir()."""
         try:
-            path = pydantic.TypeAdapter(pydantic.DirectoryPath).validate_python(value)
-        except pydantic.ValidationError as e:
-            self.fail(message="\n".join(_["msg"] for _ in e.errors()), param=param, ctx=ctx)
+            path = pathlib.Path(value)
+        except TypeError:
+            self.fail(message="Not a valid path", param=param, ctx=ctx)
 
         if self.exists and not path.exists():
             self.fail(message="Directory does not exist", param=param, ctx=ctx)
 
-        if self.make:
+        if not path.exists() and self.make:
+            _LOG.warning(f"The directory '{path}' does not yet exist, creating it..")
             path.mkdir(parents=True, exist_ok=True)
 
         return path.resolve()
@@ -159,7 +159,7 @@ def _parse_syncer_configuration(
             self.fail(message=f"Syncer definition file does not exist at '{definition_spec}'.", param=param, ctx=ctx)
 
         except toml.TomlDecodeError:
-            log.debug(f"Syncer definition file '{definition_spec}' is invalid TOML.", exc_info=True)
+            _LOG.debug(f"Syncer definition file '{definition_spec}' is invalid TOML.", exc_info=True)
             self.fail(message=f"Syncer definition file '{definition_spec}' is invalid TOML.", param=param, ctx=ctx)
 
         return options
@@ -175,7 +175,7 @@ def convert(self, value: Any, param: Optional[click.Parameter], ctx: Optional[cl
         protocol, _, definition_spec = value.partition("://")
 
         # fmt: off
-        log.debug(f"Registering syncer: {protocol.lower()}")
+        _LOG.debug(f"Registering syncer: {protocol.lower()}")
         syncer_base_dir = CS_TOOLS_PKG_DIR / "sync" / protocol
         syncer_manifest = base.SyncerManifest.model_validate_json(syncer_base_dir.joinpath("MANIFEST.json").read_text())
         syncer_options  = self._parse_syncer_configuration(definition_spec, param=param, ctx=ctx)
@@ -186,7 +186,7 @@ def convert(self, value: Any, param: Optional[click.Parameter], ctx: Optional[cl
         if issubclass(SyncerClass, base.DatabaseSyncer) and self.models is not None:
             syncer_options["models"] = self.models
 
-        log.info(f"Initializing syncer: {SyncerClass}")
+        _LOG.info(f"Initializing syncer: {SyncerClass}")
         syncer = SyncerClass(**syncer_options)
 
         # CLEAN UP DATABASE RESOURCES.
@@ -236,7 +236,7 @@ def convert(self, value: Any, param: Optional[click.Parameter], ctx: Optional[cl
             )
 
         except Exception:
-            log.debug(f"Could not coerce all values to '{self.type_caster}', {values}", exc_info=True)
+            _LOG.debug(f"Could not coerce all values to '{self.type_caster}', {values}", exc_info=True)
             self.fail(message=f"Could not coerce all values to '{self.type_caster}', {values}", param=param, ctx=ctx)
 
         return values

cs_tools/cli/tools/bulk-deleter/app.py

@@ -3,6 +3,7 @@
 import collections
 import datetime as dt
 import logging
+import pathlib
 import threading
 import time
 
@@ -49,9 +50,10 @@ def downstream(
         help="protocol and path for options to pass to the syncer",
         rich_help_panel="Syncer Options",
     ),
-    directory: custom_types.Directory = typer.Option(
+    directory: pathlib.Path = typer.Option(
         None,
-        help="folder/directory to export TML objects to",
+        help="Folder/directory to export TML objects to",
+        click_type=custom_types.Directory(exists=False, make=True),
         rich_help_panel="TML Export Options",
     ),
     export_only: bool = typer.Option(
@@ -60,7 +62,7 @@ def downstream(
         help="export all tagged content, but don't remove it from ThoughtSpot",
         rich_help_panel="TML Export Options",
     ),
-    org_override: str = typer.Option(None, "--org", help="The org to import TML to."),
+    org_override: str = typer.Option(None, "--org", help="The Org to switch to before performing actions."),
 ) -> _types.ExitCode:
     """
     Delete all downstream dependencies of an object.
@@ -215,9 +217,10 @@ def from_tag(
     tag_name: str = typer.Option(..., "--tag", help="case sensitive name to tag stale objects with"),
     tag_only: bool = typer.Option(False, "--tag-only", help="delete only the tag itself, not the objects"),
     no_prompt: bool = typer.Option(False, "--no-prompt", help="disable the confirmation prompt"),
-    directory: custom_types.Directory = typer.Option(
+    directory: pathlib.Path = typer.Option(
         None,
-        help="folder/directory to export TML objects to",
+        help="Folder/directory to export TML objects to",
+        click_type=custom_types.Directory(exists=False, make=True),
         rich_help_panel="TML Export Options",
     ),
     export_only: bool = typer.Option(
@@ -226,13 +229,17 @@ def from_tag(
         help="export all tagged content, but don't remove it from ThoughtSpot",
         rich_help_panel="TML Export Options",
     ),
+    org_override: str = typer.Option(None, "--org", help="The Org to switch to before performing actions."),
 ) -> _types.ExitCode:
     """Delete content with the identified --tag."""
     if export_only and directory is None:
         raise typer.BadParameter("You must provide a directory to export to when using --export-only.")
 
     ts = ctx.obj.thoughtspot
 
+    if ts.session_context.thoughtspot.is_orgs_enabled and org_override is not None:
+        ts.switch_org(org_id=org_override)
+
     try:
         c = workflows.metadata.fetch_one(tag_name, "TAG", http=ts.api)
         _ = utils.run_sync(c)
@@ -338,9 +345,10 @@ def from_tabular(
     ),
     deletion: str = typer.Option(..., help="directive to find content to delete", rich_help_panel="Syncer Options"),
     no_prompt: bool = typer.Option(False, "--no-prompt", help="disable the confirmation prompt"),
-    directory: custom_types.Directory = typer.Option(
+    directory: pathlib.Path = typer.Option(
         None,
-        help="folder/directory to export TML objects to",
+        help="Folder/directory to export TML objects to",
+        click_type=custom_types.Directory(exists=False, make=True),
         rich_help_panel="TML Export Options",
     ),
     export_only: bool = typer.Option(

cs_tools/cli/tools/bulk_sharing/app.py

@@ -74,10 +74,14 @@ def from_tag(
         help="The level of access to give to all listed principals.",
     ),
     no_prompt: bool = typer.Option(False, "--no-prompt", help="disable the confirmation prompt"),
+    org_override: str = typer.Option(None, "--org", help="The Org to switch to before performing actions."),
 ) -> _types.ExitCode:
     """Share content with the identified --tag."""
     ts = ctx.obj.thoughtspot
 
+    if ts.session_context.thoughtspot.is_orgs_enabled and org_override is not None:
+        ts.switch_org(org_id=org_override)
+
     try:
         c = workflows.metadata.fetch_one(tag_name, "TAG", http=ts.api)
         _ = utils.run_sync(c)

cs_tools/cli/tools/git/branches.py

@@ -72,7 +72,7 @@ def branches_commit(
         False, "--delete-aware", help="Deletes content in the GitHub repository if it is not present in this commit."
     ),
     log_errors: bool = typer.Option(False, "--log-errors", help="Log API errors to the console."),
-    org_override: str = typer.Option(None, "--org", help="The org to commit objects from."),
+    org_override: str = typer.Option(None, "--org", help="The Org to switch to before performing actions."),
     # === DEPRECATED ===
     branch_override: str = typer.Option(
         None,
@@ -177,7 +177,7 @@ def branches_validate(
     ctx: typer.Context,
     source: str = typer.Option(..., "--source-branch", help="The source branch to merge from."),
     target: str = typer.Option(..., "--target-branch", help="The target branch to merge into."),
-    org_override: str = typer.Option(None, "--org", help="The source Org to use when comparing branches."),
+    org_override: str = typer.Option(None, "--org", help="The Org to switch to before performing actions."),
 ):
     """Validates that your GitHub branches can be merged."""
     ts = ctx.obj.thoughtspot
@@ -231,7 +231,7 @@ def branches_deploy(
         help="Whether to accept any errors during the DEPLOY.",
         rich_help_panel="TML Deploy Options",
     ),
-    org_override: str = typer.Option(None, "--org", help="The org to deploy TML to."),
+    org_override: str = typer.Option(None, "--org", help="The Org to switch to before performing actions."),
     log_errors: bool = typer.Option(False, "--log-errors", help="Log TML errors to the console."),
 ):
     """Pulls from a branch in a GitHub repository to ThoughtSpot."""
@@ -307,7 +307,7 @@ def commits_search(
         "--branch-name",
         help="The name of the branch to search.",
     ),
-    org_override: str = typer.Option(None, "--org", help="The org to search commit from."),
+    org_override: str = typer.Option(None, "--org", help="The Org to switch to before performing actions."),
 ):
     """Searches for the commits for the given metadata ID."""
     ts = ctx.obj.thoughtspot
@@ -337,7 +337,7 @@ def commit_revert(
         "--branch-name",
         help="The name of the branch to search.",
     ),
-    org_override: str = typer.Option(None, "--org", help="The org to revert the commit from."),
+    org_override: str = typer.Option(None, "--org", help="The Org to switch to before performing actions."),
 ):
     """Searches for the commits for the given metadata ID."""
     ts = ctx.obj.thoughtspot

cs_tools/cli/tools/git/config.py

@@ -29,7 +29,7 @@ def config_create(
     access_token: str = typer.Option(..., help="A persanal access token with access to the GitHub repository."),
     commit_branch: str = typer.Option(..., help="The name of the branch to save TML to."),
     config_branch: str = typer.Option(..., help="The name of the branch to use for GUID mapping."),
-    org_override: str = typer.Option(None, "--org", help="The org to use."),
+    org_override: str = typer.Option(None, "--org", help="The default Org to switch to when issuing commands."),
 ) -> _types.ExitCode:
     """Creates a GitHub configuration for an org."""
     ts = ctx.obj.thoughtspot
@@ -69,7 +69,7 @@ def config_create(
 @depends_on(thoughtspot=ThoughtSpot())
 def config_update(
     ctx: typer.Context,
-    org_override: str = typer.Option(None, "--org", help="The org to use."),
+    org_override: str = typer.Option(None, "--org", help="The default Org to switch to when issuing commands."),
     repository_url: str = typer.Option(None, help="The GitHub repository URL to use."),
     username: str = typer.Option(None, help="The username of a user with access to the GitHub repository."),
     access_token: str = typer.Option(None, help="A persanal access token with access to the GitHub repository."),
@@ -97,7 +97,10 @@ def config_update(
 
 @app.command(name="search")
 @depends_on(thoughtspot=ThoughtSpot())
-def config_search(ctx: typer.Context, org_override: str = typer.Option(None, "--org", help="The org to use.")):
+def config_search(
+    ctx: typer.Context,
+    org_override: str = typer.Option(None, "--org", help="The default Org to switch to when issuing commands."),
+):
     """Searches for configurations."""
     ts = ctx.obj.thoughtspot