Merge branch 'dev'

Author: boonhapus

.gitignore

@@ -139,3 +139,6 @@ dmypy.json
 dist/**/*.whl
 dist/**/*.zip
 dist/**/*.tar.gz
+
+tmlout/*
+tmlvalidate/*

cs_tools/_version.py

@@ -1 +1 @@
-__version__ = '1.3.2'
+__version__ = '1.3.3'

cs_tools/api/_rest_api_v1.py

@@ -12,6 +12,7 @@
     _Security,
     _Session,
     Data,
+    Group,
     Logs,
     Metadata,
     Security,
@@ -32,7 +33,8 @@ def _secure_for_log(kw) -> Dict[str, Any]:
     except TypeError:
         secure = copy.deepcopy({k: v for k, v in kw.items() if k not in ('file', 'files')})
 
-    return secure.get('data', {}).pop('password', None)
+    secure.get('data', {}).pop('password', None)
+    return secure
 
 
 class _RESTAPIv1:
@@ -56,6 +58,7 @@ def __init__(self, config, ts):
 
         # public API endpoints
         self.data = Data(self)
+        self.group = Group(self)
         self.metadata = Metadata(self)
         self.security = Security(self)
         self.user = User(self)
@@ -75,6 +78,7 @@ def request(
         endpoint: str,
         *,
         privacy: str='public',
+        timeout: int=-1,
         **kw
     ) -> httpx.Response:
         """
@@ -107,6 +111,10 @@ def request(
         **kw
             passed into the httpx.request call
         """
+        # "httpx.Client.timeout = None" has meaning, -1 does not.
+        if timeout == -1:
+            timeout = self._http.timeout
+
         if httpx.URL(endpoint).is_relative_url:
             _privacy = {
                 # IF NOT FOUND IN THIS MAPPING, THEN IT'S AN UNDOCUMENTED API
@@ -124,7 +132,7 @@ def request(
         log.debug(f'{method} >> {endpoint} with data:\n\tkwargs={_secure_for_log(kw)}')
 
         meth = getattr(self._http, method.lower())
-        r = meth(endpoint, **kw)
+        r = meth(endpoint, **kw, timeout=timeout)
         r.raise_for_status()
         log.debug(f'<< HTTP: {r.status_code}')
 

cs_tools/api/middlewares/__init__.py

@@ -1,4 +1,5 @@
 from .connection import ConnectionMiddleware
+from .group import GroupMiddleware
 from .metadata import MetadataMiddleware
 from .pinboard import PinboardMiddleware
 from .answer import AnswerMiddleware

cs_tools/api/middlewares/group.py

@@ -0,0 +1,24 @@
+from typing import Any, Dict, List, Union
+import logging
+
+from pydantic import validate_arguments
+
+log = logging.getLogger(__name__)
+
+
+class GroupMiddleware:
+    """
+    Functions to simplify using the group API
+    """
+    def __init__(self, ts):
+        self.ts = ts
+
+    @validate_arguments
+    def get_group_id(self, name: str):
+        """
+        Returns the GUID for a group with the given name.
+        :param name: Name of the group, like 'Administrator'
+        """
+        r = self.ts.api.group.get_group(name=name)
+        info = r.json()
+        return info['header']['id']

cs_tools/api/middlewares/metadata.py

@@ -5,7 +5,6 @@
 
 from cs_tools.data.enums import (
     DownloadableContent, GUID, MetadataCategory, MetadataObject, MetadataObjectSubtype,
-    PermissionType
 )
 from cs_tools.errors import ContentDoesNotExist
 from cs_tools.util import chunks
@@ -302,35 +301,54 @@ def get_edoc_object_list(self, guids: List[GUID]) -> List[Dict[str,str]]:
         return mapped_guids
 
     @validate_arguments
-    def get_object_ids_with_tags(self, tags: List[str]) -> List[Dict[str,str]]:
+    def get_object_ids_with_tags_or_author(
+            self, tags: List[str] = None,
+            author: GUID = None,
+            ignore_datasources: bool = False
+    ) -> List[Dict[str,str]]:
         """
-        Gets a list of IDs for the associated tag and returns as a list of object ID to type mapping.
+        Gets a list of IDs for the associated tag and/or author and returns as a list of object ID to type mapping.
+        Either the author or the tag must be specified.  If both are specified, it will be the content owned by that
+        person with the given tag.
 
         The return format looks like:
         [
             {"id": "0291f1cd-5f8e-4d96-80e2-e5ef1aa6c44f", "type":"QUESTION_ANSWER_BOOK"},
             {"id": "4bcaadb4-031a-4afd-b159-2c0c0f194c42", "type":"PINBOARD_ANSWER_BOOK"}
         ]
         :param tags: The list of tags to get ids for.
+        :param author: The author of the content.
+        :param ignore_datasources: If true, will ignore data sources.
         """
 
+        # testing - remove later
         object_ids = []
         for metadata_type in DownloadableContent:
+            if ignore_datasources and metadata_type == DownloadableContent.data_source:
+                continue
+
             offset = 0
 
             while True:
-                r = self.ts.api._metadata.list(type=metadata_type.value, batchsize=500, offset=offset, tagname=tags)
+                r = self.ts.api._metadata.list(type=metadata_type.value, batchsize=500,
+                                               offset=offset, tagname=tags, authorguid=author)
                 data = r.json()
                 offset += len(data)
 
                 for metadata in data['headers']:
-                    object_ids.append(metadata["id"])
+
+                    if author and metadata['author'] == author:  # workaround for 7.1/7.2
+                        object_ids.append(metadata["id"])
+                    elif not author:  # no author specified, so just add it.
+                        object_ids.append(metadata["id"])
+                    # else just ignore it.
 
                 if data['isLastBatch']:
                     break
 
         return list(set(object_ids))  # might have been duplicates
 
+
     @classmethod
     @validate_arguments
     def map_subtype_to_type(self, subtype: Union[str, None]) -> str:
@@ -344,7 +362,7 @@ def map_subtype_to_type(self, subtype: Union[str, None]) -> str:
 
         return subtype
 
-    def _lookup_geo_config(self, column_details) -> str:
+    def _lookup_geo_config(self, column_details) -> Union[str, None]:
         try:
             config = column_details['geoConfig']
         except KeyError:
@@ -362,7 +380,7 @@ def _lookup_geo_config(self, column_details) -> str:
 
         return 'Unknown'
 
-    def _lookup_calendar_guid(self, column_details) -> str:
+    def _lookup_calendar_guid(self, column_details) -> Union[str, None]:
         try:
             ccal_guid = column_details['calendarTableGUID']
         except KeyError:
@@ -375,12 +393,13 @@ def _lookup_calendar_guid(self, column_details) -> str:
 
         return self.cache['calendar_type'][ccal_guid]
 
-    def _lookup_currency_type(self, column_details) -> str:
+    def _lookup_currency_type(self, column_details) -> Union[str, None]:
         try:
             currency_info = column_details['currencyTypeInfo']
         except KeyError:
             return None
 
+        name = None
         if currency_info['setting'] == 'FROM_USER_LOCALE':
             name = 'Infer From Browser'
         elif currency_info['setting'] == 'FROM_ISO_CODE':
@@ -396,3 +415,41 @@ def _lookup_currency_type(self, column_details) -> str:
                 name = self.cache['currency_type'][g]
 
         return name
+
+    @validate_arguments
+    def objects_exist(self, metadata_type: MetadataObject, guids: List[GUID]) -> Dict[GUID, bool]:
+        """
+        Checks if the list of objects exist.
+        :param metadata_type: The type to check for.  Must do one at a time.
+        :param guids: The list of GUIDs to check.
+        :return: A map of GUID to boolean, where True == it exists.
+        """
+        r = self.ts.api.metadata.list_object_headers(type=metadata_type, fetchids=guids)
+        content = r.json()
+
+        # The response is a list of objects that only include the ones that exist.  So check each GUID and add to the
+        # map.
+        returned_ids = [obj.get("id") for obj in content]
+        existence = {}
+        for guid in guids:
+            existence[guid] = guid in returned_ids
+
+        return existence
+
+    @classmethod
+    @validate_arguments
+    def tml_type_to_metadata_object(cls, tml_type: str) -> Union[MetadataObject, None]:
+        """
+        Converts a tml type (e.g. "worksheet") to a MetadataObject type, (e.g. MetadataObject.logical_table)
+        :param tml_type:  The TML type, such as None
+        """
+        mapping = {
+            "table": MetadataObject.logical_table,
+            "view": MetadataObject.logical_table,
+            "worksheet": MetadataObject.logical_table,
+            "pinboard": MetadataObject.pinboard,
+            "liveboard": MetadataObject.pinboard,
+            "answer": MetadataObject.saved_answer,
+        }
+
+        return mapping.get(tml_type, None)

cs_tools/api/middlewares/tql.py

@@ -50,7 +50,7 @@ def query(
         sample: int = 50,
         database: str = None,
         schema_: Annotated[str, Field(alias='schema')] = 'falcon_default_schema',
-        http_timeout: int = 5.0
+        http_timeout: int = 60.0
     ) -> Tuple[List[Dict[str, str]], List[Dict[str, Any]]]:
         """
 
@@ -103,7 +103,7 @@ def command(
         database: str = None,
         schema_: str = 'falcon_default_schema',
         raise_errors: bool = False,
-        http_timeout: int = 5.0
+        http_timeout: int = 60.0
     ) -> List[Dict[str, Any]]:
         """
         """
@@ -143,7 +143,7 @@ def script(
         fp: pathlib.Path,
         *,
         raise_errors: bool = False,
-        http_timeout: int = 5.0
+        http_timeout: int = 60.0
     ) -> List[Dict[str, Any]]:
         """
         """

cs_tools/api/middlewares/tsload.py

@@ -117,7 +117,6 @@ def upload(
         self,
         fd: Union[BufferedIOBase, TextIOWrapper, _TemporaryFileWrapper],
         *,
-        ignore_node_redirect: bool = False,
         database: str,
         table: str,
         schema_: str = 'falcon_default_schema',
@@ -134,6 +133,9 @@ def upload(
         escape_character: str = '"',
         enclosing_character: str = '"',
         flexible: bool = False,
+        # not related to Remote TSLOAD API
+        ignore_node_redirect: bool = False,
+        http_timeout: int = 60.0
     ) -> List[Dict[str, Any]]:
         """
         Load a file via tsload on a remote server.
@@ -210,7 +212,7 @@ def upload(
         }
 
         try:
-            r = self.ts.api.ts_dataservice.load_init(flags)
+            r = self.ts.api.ts_dataservice.load_init(flags, timeout=http_timeout)
         except Exception as e:
             raise TSLoadServiceUnreachable(
                 f'[red]something went wrong trying to access tsload service: {e}[/]'
@@ -238,7 +240,7 @@ def upload(
         if not ignore_node_redirect:
             self._check_for_redirect_auth(data['cycle_id'])
 
-        self.ts.api.ts_dataservice.load_start(data['cycle_id'], fd=fd)
+        self.ts.api.ts_dataservice.load_start(data['cycle_id'], fd=fd, timeout=http_timeout)
         self.ts.api.ts_dataservice.load_commit(data['cycle_id'])
         return data['cycle_id']
 
@@ -276,13 +278,13 @@ def status(
             if not wait_for_complete:
                 break
 
-            if data['internal_stage'] == 'COMMITTING':
-                pass
-            elif data['internal_stage'] == 'DONE':
+            if data['internal_stage'] == 'DONE':
                 break
-            elif data['status']['message'] != 'OK':
+            
+            if data['status']['message'] != 'OK':
                 break
 
+            log.debug(f'data load status:\n{data}')
             time.sleep(1)
 
         return data

cs_tools/api/middlewares/user.py

@@ -97,3 +97,17 @@ def get(
             user = user[0]
 
         return user
+
+    @validate_arguments
+    def get_guid(
+        self,
+        name: str
+    ) -> Union[GUID, None]:
+        """
+        Returns the GUID for a user or None if the user wasn't found.
+        :param name: The user name, e.g. somebody@somecompany.com
+        :return: GUID or None
+        """
+        r = self.ts.api.user.get(name=name)
+        user = r.json()
+        return user['header']['id']

cs_tools/api/models/__init__.py

@@ -1,6 +1,7 @@
 from .ts_dataservice import TSDataService
 from .connection import _Connection
 from .dependency import _Dependency
+from .group import Group
 from .periscope import _Periscope
 from .metadata import _Metadata, Metadata
 from .security import _Security, Security