Merge branch 'dev'

Author: boonhapus

.gitignore

@@ -6,6 +6,7 @@ __pycache__/
 
 # Mac stuff
 .DS_Stores/
+.DS_Store
 
 # C extensions
 *.so

MANIFEST.in

@@ -1 +1 @@
-include cs_tools/tools/**/static/*
+recursive-include cs_tools/tools/**/static *

README.md

@@ -36,7 +36,7 @@ From your command line:
 $ cd $HOME
 $ python3 -m venv .cs_tools-dev
 $ source .cs_tools-dev/bin/activate
-$ pip install -e git+https://github.com/thoughtspot/cs_tools.git
+$ pip install -e .
 ```
 
 That's it!

cs_tools/_version.py

@@ -1 +1 @@
-__version__ = '1.0.3'
+__version__ = '1.1.0'

cs_tools/api.py

@@ -14,7 +14,6 @@
 from cs_tools.models.user import User
 from cs_tools.schema.user import User as UserSchema
 from cs_tools.errors import CertificateVerifyFailure
-from cs_tools.const import APP_DIR
 
 
 log = logging.getLogger(__name__)
@@ -25,7 +24,6 @@ class ThoughtSpot:
     """
     def __init__(self, ts_config):
         self.config = ts_config
-        self._setup_logging()
 
         # set up our session
         # NOTE: base_url is a valid parameter for httpx.Client
@@ -51,31 +49,6 @@ def __init__(self, ts_config):
         self._security = _Security(self)
         self._session = _Session(self)
 
-    def _clean_logs(self, now):
-        logs_dir = APP_DIR / 'logs'
-        logs_dir.mkdir(parents=True, exist_ok=True)
-
-        # keep only the last 25 logfiles
-        lifo = sorted(logs_dir.iterdir(), reverse=True)
-
-        for idx, log in enumerate(lifo):
-            if idx > 25:
-                log.unlink()
-
-    def _setup_logging(self):
-        logging.getLogger('urllib3').setLevel(logging.ERROR)
-
-        now = dt.datetime.now().strftime('%Y-%m-%dT%H_%M_%S')
-        self._clean_logs(now)
-
-        logging.basicConfig(
-            filename=f'{APP_DIR}/logs/{now}.log',
-            format='[%(levelname)s - %(asctime)s] '
-                   '[%(name)s - %(module)s.%(funcName)s %(lineno)d] '
-                   '%(message)s',
-            level=self.config.logging.level
-        )
-
     @property
     def host(self):
         """

cs_tools/cli.py

@@ -1,7 +1,11 @@
+import datetime as dt
 import pathlib
+import logging
 import shutil
+import logging
 
 from typer import Argument as A_, Option as O_
+from rich.logging import RichHandler
 import pydantic
 import click
 import typer
@@ -11,6 +15,13 @@
 from cs_tools.helpers.loader import _gather_tools
 from cs_tools.util.algo import deep_update
 from cs_tools.settings import TSConfig
+from cs_tools.const import APP_DIR
+
+
+log = logging.getLogger(__name__)
+
+
+log = logging.getLogger(__name__)
 
 
 app = typer.Typer(
@@ -105,7 +116,7 @@ def export(
     app_dir = pathlib.Path(typer.get_app_dir('cs_tools'))
     log_dir = app_dir / 'logs'
 
-    for log in log_dir.glob('*.tml'):
+    for log in log_dir.iterdir():
         shutil.copy(log, save_path)
 
 
@@ -227,6 +238,18 @@ def delete(
     console.print(f'removed cluster configuration file "{name}"')
 
 
+def _clean_logs(now):
+    logs_dir = APP_DIR / 'logs'
+    logs_dir.mkdir(parents=True, exist_ok=True)
+
+    # keep only the last 25 logfiles
+    lifo = sorted(logs_dir.iterdir(), reverse=True)
+
+    for idx, log in enumerate(lifo):
+        if idx > 25:
+            log.unlink()
+
+
 def run():
     """
     Entrypoint into cs_tools.
@@ -236,10 +259,28 @@ def run():
     app.add_typer(cfg_app, name='config')
     app.add_typer(log_app, name='logs')
 
+    # SETUP LOGGING
+    logging.getLogger('urllib3').setLevel(logging.ERROR)
+
+    now = dt.datetime.now().strftime('%Y-%m-%dT%H_%M_%S')
+    _clean_logs(now)
+
+    logging.basicConfig(
+        filename=f'{APP_DIR}/logs/{now}.log',
+        format='[%(levelname)s - %(asctime)s] '
+               '[%(name)s - %(module)s.%(funcName)s %(lineno)d] '
+               '%(message)s',
+        level='DEBUG'
+    )
+
     try:
         app(prog_name='cs_tools')
     except Exception as e:
+        log.debug('whoopsie, something went wrong!', exc_info=True)
+
         if hasattr(e, 'warning'):
             e = e.warning
+        else:
+            e = f'{type(e).__name__}: {e}'
 
-        console.print(f'[error]{e}')
+        log.exception(f'[error]{e}')

cs_tools/models/_base.py

@@ -1,4 +1,5 @@
 import logging
+import copy
 import json
 
 import httpx
@@ -23,6 +24,9 @@ def base_url(self):
         host = self.config.thoughtspot.host
         port = self.config.thoughtspot.port
 
+        if not host.startswith('http'):
+            host = f'https://{host}'
+
         if port:
             port = f':{port}'
         else:
@@ -45,9 +49,13 @@ def _request(self, method, url, *args, **kwargs) -> httpx.Response:
         # sigh/
 
         # don't log the password
-        secure = kwargs.copy()
-        secure.pop('password', None)
-        log.debug(f'>> {method} to {url} with data:\nargs={args}\nkwargs={secure}')
+        try:
+            secure = copy.deepcopy(kwargs)
+        except TypeError:
+            secure = copy.deepcopy({k: v for k, v in kwargs.items() if k not in ('file', 'files')})
+
+        secure.get('data', {}).pop('password', None)
+        log.debug(f'>> {method} to {url} with data:\n\targs={args}\n\tkwargs={secure}')
 
         r = self.http.request(method, url, *args, **kwargs)
         log.debug(f'<< {r.status_code} from {url}')

cs_tools/models/metadata.py

@@ -2,8 +2,10 @@
 import logging
 import enum
 
+import pydantic
 import httpx
 
+from cs_tools.util.swagger import to_array
 from cs_tools.settings import APIParameters
 from cs_tools.models import TSPrivate, TSPublic
 
@@ -70,7 +72,7 @@ class ListVizHeadersParameters(APIParameters):
 
 class ListObjectHeadersParameters(APIParameters):
     type: Union[MetadataObject, None] = MetadataObject.PINBOARD_ANSWER_BOOK
-    subtypes: LogicalTableSubtype = None
+    subtypes: List[LogicalTableSubtype] = None
     category: MetadataCategory = MetadataCategory.ALL
     sort: SortOrder = SortOrder.DEFAULT
     sortascending: bool = None
@@ -83,6 +85,10 @@ class ListObjectHeadersParameters(APIParameters):
     fetchids: str = None
     auto_created: bool = None
 
+    @pydantic.validator('subtypes')
+    def stringify_the_array(cls, v):
+        return to_array([_.value for _ in v])
+
 
 class ListParameters(ListObjectHeadersParameters):
     ownertypes: LogicalTableSubtype = None
@@ -106,6 +112,21 @@ class DetailParameters(APIParameters):
     doUpdate: bool = True
 
 
+class ListColumnParameters(APIParameters):
+    id: str
+    showhidden: bool = False
+
+
+class DeleteParameters(APIParameters):
+    type: MetadataObject = None
+    id: str
+    includedisabled: bool = False
+
+    @pydantic.validator('id', pre=True)
+    def stringify_the_array(cls, v) -> str:
+        return to_array(v)
+
+
 #
 
 class Metadata(TSPublic):
@@ -167,16 +188,32 @@ def list(self, **parameters) -> httpx.Response:
 
     def listas(self, **parameters) -> httpx.Response:
         """
-        TODO
+        List of metadata objects in the repository as seen by a User/Group.
         """
         p = ListAsParameters(**parameters)
         r = self.get(f'{self.base_url}/listas', params=p.json())
         return r
 
     def detail(self, guid, **parameters) -> httpx.Response:
         """
-        TODO
+        Detail of a metadata object in the repository.
         """
         p = DetailParameters(id=guid, **parameters)
         r = self.get(f'{self.base_url}/detail/{guid}', params=p.json())
         return r
+
+    def delete(self, **parameters) -> httpx.Response:
+        """
+        Delete metadata object(s) from the repository.
+        """
+        p = DeleteParameters(**parameters)
+        r = self.post(f'{self.base_url}/delete', data=p.json())
+        return r
+
+    def list_columns(self, guid, **parameters) -> httpx.Response:
+        """
+        Get list of all logical columns of a given logical table.
+        """
+        p = ListColumnParameters(id=guid, **parameters)
+        r = self.get(f'{self.base_url}/listcolumns/{guid}', params=p.json())
+        return r

cs_tools/models/periscope.py

@@ -25,6 +25,9 @@ def base_url(self):
         host = self.config.thoughtspot.host
         port = self.config.thoughtspot.port
 
+        if not host.startswith('http'):
+            host = f'https://{host}'
+
         if port:
             port = f':{port}'
         else:

cs_tools/models/security.py

@@ -1,19 +1,25 @@
-import enum
+from typing import List, Dict
 import pydantic
 import httpx
-from typing import List
+import enum
 
+from cs_tools.util.swagger import to_array
 from cs_tools.settings import APIParameters
 from cs_tools.models import TSPrivate
 
 
+import logging
+log = logging.getLogger(__name__)
+
+
 class ObjectType(enum.Enum):
+    LOGICAL_COLUMN = "LOGICAL_COLUMN"              # table-, worksheet-, view-columns
     LOGICAL_TABLE = "LOGICAL_TABLE"                # tables, worksheets, views
     QUESTION_ANSWER_BOOK = "QUESTION_ANSWER_BOOK"  # answer
     PINBOARD_ANSWER_BOOK = "PINBOARD_ANSWER_BOOK"  # pinboard
 
 
-class SharePermission(str, enum.Enum):
+class SharePermission(enum.Enum):
     MODIFY = "MODIFY"
     NO_ACCESS = "NO_ACCESS"
     READ_ONLY = "READ_ONLY"
@@ -23,27 +29,51 @@ def __str__(self):
 
 
 class ShareParameters(APIParameters):
+    # NOTE: FORMAT OF .permission
+    #
+    # {
+    #     'permissions': {
+    #         guid: {
+    #             'shareMode': permission
+    #         }
+    #     }
+    # }
+    #
     type: ObjectType = None
     id: str = None
-    # permission: Dict[str, Dict[str, Dict[str, SharePermission]]] = {}
-    permission: str = ""
+    permission: Dict[str, Dict[str, Dict[str, SharePermission]]]
+    # permission: str = ""
     emailshares: List[pydantic.EmailStr] = []
     notify: bool = True
     message: str = ""
 
-    # TODO is it possible to use util.to_array()?
     @pydantic.validator('id', pre=True)
-    def _flatten_id_list(cls, v):
-        if not isinstance(id, str):
-            v = ",".join(v)
-        return f"[{v}]"
+    def stringify_the_array(cls, v) -> str:
+        return to_array(v)
 
     @pydantic.validator('permission', pre=True)
-    def _inject_share_mode(cls, v):
+    def inject_share_mode(cls, v):
+        if 'permissions' in v:
+            v = v['permissions']
+
         new_v = {}
-        for group_id, permission in v.items():
-            new_v[group_id] = {'shareMode': str(permission)}
-        return str({'permissions': new_v})
+
+        for group_guid, permission in v.items():
+            if 'shareMode' not in permission:
+                permission = {'shareMode': permission}
+
+            new_v[group_guid] = permission
+
+        return {'permissions': new_v}
+
+
+class DefinedPermissionParameters(APIParameters):
+    type: str
+    id: str
+
+    @pydantic.validator('id', pre=True)
+    def stringify_the_array(cls, v):
+        return to_array(v)
 
 
 class _Security(TSPrivate):
@@ -63,5 +93,14 @@ def share(self, **parameters) -> httpx.Response:
         List of metadata objects in the repository.
         """
         p = ShareParameters(**parameters)
+        # print(p.json())
         r = self.post(f'{self.base_url}/share', data=p.json())
         return r
+
+    def defined_permission(self, **parameters) -> httpx.Response:
+        """
+        Get defined permissions information for a given list of objects
+        """
+        p = DefinedPermissionParameters(**parameters)
+        r = self.post(f'{self.base_url}/definedpermission', data=p.json())
+        return r