Bugs and fixes for Searchable

cs_tools/__project__.py

@@ -1,4 +1,4 @@
-__version__ = "1.6.3"
+__version__ = "1.6.4"
 __docs__ = "https://thoughtspot.github.io/cs_tools/"
 __repo__ = "https://github.com/thoughtspot/cs_tools"
 __help__ = f"{__repo__}/discussions/"

cs_tools/api/client.py

@@ -430,6 +430,19 @@ def groups_search(
         timeout = options.pop("timeout", httpx.USE_CLIENT_DEFAULT)
         return self.post("api/rest/2.0/groups/search", headers=headers, timeout=timeout, json=options)
 
+    @pydantic.validate_call(validate_return=True, config=validators.METHOD_CONFIG)
+    @_transport.CachePolicy.mark_cacheable
+    def groups_search_v1(self, **options: Any) -> Awaitable[httpx.Response]:  # noqa: ARG002
+        """Get a list of ThoughtSpot groups with v1 endpoint."""
+        return self.get("callosum/v1/tspublic/v1/group")
+
+    @pydantic.validate_call(validate_return=True, config=validators.METHOD_CONFIG)
+    @_transport.CachePolicy.mark_cacheable
+    def group_list_users(self, group_guid: _types.ObjectIdentifier, **options: Any) -> Awaitable[httpx.Response]:  # noqa: ARG002
+        """Get a list of ThoughtSpot users in a group with v1 endpoint."""
+        r = self.get(f"callosum/v1/tspublic/v1/group/{group_guid}/users")
+        return r
+
     @pydantic.validate_call(validate_return=True, config=validators.METHOD_CONFIG)
     def groups_create(self, **options: Any) -> Awaitable[httpx.Response]:
         """Create a ThoughtSpot group."""

cs_tools/cli/tools/archiver/app.py

@@ -41,6 +41,19 @@ def _tick_tock(task: px.WorkTask) -> None:
         task.advance(step=1)
 
 
+def get_group_users(guids: list[_types.ObjectIdentifier], users_profiles: list[dict]):
+    users = []
+    for guid in guids:
+        for user in users_profiles:
+            if guid in user["assignedGroups"]:
+                users.append(user["header"])
+
+            if guid in user["inheritedGroups"]:
+                users.append(user["header"])
+
+    return users
+
+
 app = AsyncTyper(
     help="""
     Manage stale answers and liveboards within your platform.
@@ -156,7 +169,7 @@ def identify(
 
             SEARCH_TOKENS = (
                 # SELECT TS: BI ACTIVITY ON ANY ACTIVITY WITH A QUERY
-                "[query text] != '{null}' "
+                # "[query text] != '{null}' "
                 # FILTER OUT AD-HOC SEARCH
                 "[user action] != [user action].answer_unsaved "
                 # FILTER OUT POTENTIAL DATA QUALITY ISSUES ? (just here for safety~)
@@ -185,20 +198,69 @@ def identify(
 
         with tracker["GATHER_METADATA"]:
             if only_groups or ignore_groups:
-                c = workflows.paginator(ts.api.groups_search, record_size=150_000, timeout=60 * 15)
-                d = utils.run_sync(c)
+                c = ts.api.groups_search_v1()
+                r = utils.run_sync(c)
+                d = r.json()
 
+                # Inline with V1 Group API.
                 all_groups = [
                     {
-                        "guid": group["id"],
-                        "name": group["name"],
-                        "users": group["users"],
+                        "guid": group["header"]["id"],
+                        "name": group["header"]["name"],
                     }
                     for group in d
                 ]
 
-                only_groups = [group["guid"] for group in all_groups if group["name"] in (only_groups or [])]  # type: ignore[assignment]
-                ignore_groups = [group["guid"] for group in all_groups if group["name"] in (ignore_groups or [])]  # type: ignore[assignment]
+                # c = workflows.paginator(ts.api.groups_search, record_size=150_000, timeout=60 * 15)
+                # d = utils.run_sync(c)
+
+                # all_groups = [
+                #     {
+                #         "guid": group["id"],
+                #         "name": group["name"],
+                #         "users": group["users"],
+                #     }
+                #     for group in d
+                # ]
+
+                only_groups_lst = [group["guid"] for group in all_groups if group["name"] in (only_groups or [])]  # type: ignore[assignment]
+                ignore_groups_lst = [group["guid"] for group in all_groups if group["name"] in (ignore_groups or [])]  # type: ignore[assignment]
+
+                if only_groups is not None:
+                    users_profiles: list = []
+                    for guid in only_groups_lst:
+                        c = ts.api.group_list_users(group_guid=guid)
+                        r = r = utils.run_sync(c)
+                        users = r.json()
+                        if not users:
+                            # Exception needs to be redone
+                            info = {
+                                "reason": "Group names are case sensitive. "
+                                + "You can find a group's 'Group Name' in the Admin panel.",
+                                "mitigation": "Verify the name and try again.",
+                                "type": "Group",
+                            }
+                            raise Exception(str(info)) from None
+                        users_profiles.extend(users)
+                    users_only = [user["id"] for user in get_group_users(only_groups_lst, users_profiles)]
+
+                if ignore_groups is not None:
+                    users_profiles: list = []
+                    for guid in ignore_groups_lst:
+                        c = ts.api.group_list_users(group_guid=guid)
+                        r = utils.run_sync(c)
+                        users = r.json()
+                        if not users:
+                            info = {
+                                "reason": "Group names are case sensitive. "
+                                + "You can find a group's 'Group Name' in the Admin panel.",
+                                "mitigation": "Verify the name and try again.",
+                                "type": "Group",
+                            }
+                            raise Exception(str(info)) from None
+                        users_profiles.extend(users)
+
+                    users_ignore = [user["id"] for user in get_group_users(ignore_groups_lst, users_profiles)]
 
             if ignore_tags:
                 c = workflows.metadata.fetch_all(metadata_types=["TAG"], http=ts.api)
@@ -255,18 +317,18 @@ def identify(
                 # CHECK: THE AUTHOR IS A MEMBER OF GROUPS WHOSE CONTENT SHOULD NEEDS TO INCLUDED.
                 if only_groups is not None:
                     assert isinstance(only_groups, list), "Only Groups wasn't properly transformed to an array<GUID>."
-                    checks.append(metadata_object["author_guid"] in only_groups)
+                    checks.append(metadata_object["author_guid"] in users_only)
 
                 # CHECK: THE AUTHOR IS NOT A MEMBER OF GROUPS WHOSE CONTENT SHOULD BE IGNORED.
                 if ignore_groups is not None:
                     assert isinstance(
                         ignore_groups, list
                     ), "Ignore Groups wasn't properly transformed to an array<GUID>."
-                    checks.append(metadata_object["author_guid"] not in ignore_groups)
+                    checks.append(metadata_object["author_guid"] not in users_ignore)
 
                 if ignore_tags is not None:
                     assert isinstance(ignore_tags, list), "Ignore Tags wasn't properly transformed to an array<GUID>."
-                    checks.append(any(t["id"] not in ignore_tags for t in metadata_object["tags"]))
+                    checks.append(not any(t["id"] in ignore_tags for t in metadata_object["tags"]))
 
                 if all(checks):
                     filtered.append(metadata_object)

cs_tools/cli/tools/searchable/api_transformer.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from typing import Any
 import datetime as dt
 import functools as ft
 import itertools as it
@@ -10,11 +11,13 @@
 import awesomeversion
 
 from cs_tools import _types, validators
+from cs_tools._types import TableRowsFormat
 from cs_tools.datastructures import SessionContext
 
 from . import models
 
 log = logging.getLogger(__name__)
+ArbitraryJsonFormat = list[dict[str, Any]]
 
 
 def ts_cluster(data: SessionContext) -> _types.TableRowsFormat:
@@ -111,6 +114,30 @@ def ts_group(data: list[_types.APIResult], *, cluster: _types.GUID) -> _types.Ta
     return reshaped
 
 
+def to_group_v1(data: ArbitraryJsonFormat, cluster: str) -> TableRowsFormat:
+    """Reshapes groups/search -> searchable.models.Group. Needed for V1 API."""
+    out: TableRowsFormat = []
+
+    for row in data:
+        for org_id in row["header"].get("orgIds", [0]):
+            out.append(
+                models.Group.validated_init(
+                    cluster_guid=cluster,
+                    org_id=org_id,
+                    group_guid=row["header"]["id"],
+                    group_name=row["header"]["name"],
+                    description=row["header"].get("description"),
+                    display_name=row["header"]["displayName"],
+                    sharing_visibility=row["visibility"],
+                    created=row["header"]["created"] / 1000,
+                    modified=row["header"]["modified"] / 1000,
+                    group_type=row["type"],
+                )
+            )
+
+    return [model.model_dump() for model in out]
+
+
 def ts_org_membership(data: list[_types.APIResult], *, cluster: _types.GUID) -> _types.TableRowsFormat:
     """Reshapes users/search -> searchable.models.OrgMembership."""
     reshaped: _types.TableRowsFormat = []
@@ -188,6 +215,21 @@ def ts_group_membership(data: list[_types.APIResult], *, cluster: _types.GUID) -
     return reshaped
 
 
+def to_group_membership(data: ArbitraryJsonFormat, cluster: str) -> TableRowsFormat:
+    """Reshapes {groups|users}/search -> searchable.models.GroupMembership. Needed for V1 API."""
+    out: TableRowsFormat = []
+
+    for row in data:
+        for group in row["assignedGroups"]:
+            out.append(
+                models.GroupMembership.validated_init(
+                    cluster_guid=cluster, principal_guid=row["header"]["id"], group_guid=group
+                )
+            )
+
+    return [model.model_dump() for model in out]
+
+
 def ts_group_privilege(data: list[_types.APIResult], *, cluster: _types.GUID) -> _types.TableRowsFormat:
     """Reshapes {groups|users}/search -> searchable.models.GroupPrivilege."""
     reshaped: _types.TableRowsFormat = []
@@ -205,6 +247,21 @@ def ts_group_privilege(data: list[_types.APIResult], *, cluster: _types.GUID) ->
     return reshaped
 
 
+def to_group_privilege(data: ArbitraryJsonFormat, cluster: str) -> TableRowsFormat:
+    """Reshapes {groups|users}/search -> searchable.models.GroupPrivilege. Needed for V1 API."""
+    out: TableRowsFormat = []
+
+    for row in data:
+        for privilege in row["privileges"]:
+            out.append(
+                models.GroupPrivilege.validated_init(
+                    cluster_guid=cluster, group_guid=row["header"]["id"], privilege=privilege
+                )
+            )
+
+    return [model.model_dump() for model in out]
+
+
 def ts_tag(data: list[_types.APIResult], *, cluster: _types.GUID, current_org: int) -> _types.TableRowsFormat:
     """Reshapes {groups|users}/search -> searchable.models.Tag."""
     reshaped: _types.TableRowsFormat = []

cs_tools/cli/tools/searchable/app.py

@@ -325,7 +325,7 @@ def bi_server(
         + ("" if from_date is None else f" [timestamp] >= '{from_date.strftime(SEARCH_DATA_DATE_FMT)}'")
         + ("" if to_date is None else f" [timestamp] <= '{to_date.strftime(SEARCH_DATA_DATE_FMT)}'")
         + ("" if not ts.session_context.thoughtspot.is_orgs_enabled else " [org id]")
-        + ("" if org_override is None else f" [org id] = {ts.org.guid_for(org_override)}")
+        + ("" if org_override is None else f" [org id] = {org_override}")
     )
 
     TOOL_TASKS = [
@@ -428,7 +428,11 @@ def metadata(
         tracker["ORGS_COUNT"].start()
 
         # LOOP THROUGH EACH ORG COLLECTING DATA
-        primary_org_done = False
+        if org_override is not None:
+            collect_info = True
+        else:
+            collect_info = False
+
         for org in orgs:
             tracker.title = f"Fetching Data in [fg-secondary]{org['name']}[/] (Org {org['id']})"
             seen_guids: dict[_types.APIObjectType, set[_types.GUID]] = collections.defaultdict(set)
@@ -450,29 +454,34 @@ def metadata(
                 temp.dump(models.Org.__tablename__, data=d)
 
             with tracker["TS_GROUP"]:
-                c = workflows.paginator(ts.api.groups_search, record_size=5_000, timeout=60 * 15)
-                _ = utils.run_sync(c)
+                c = ts.api.groups_search_v1()
+                r = utils.run_sync(c)
+                _ = r.json()
+
+                # commenting out calling of v2 api for CWT customer
+                # c = workflows.paginator(ts.api.groups_search, record_size=5_000, timeout=60 * 15)
+                # _ = utils.run_sync(c)
 
                 # DUMP GROUP DATA
-                d = api_transformer.ts_group(data=_, cluster=CLUSTER_UUID)
+                d = api_transformer.to_group_v1(data=_, cluster=CLUSTER_UUID)
                 temp.dump(models.Group.__tablename__, data=d)
 
                 # TODO: REMOVE AFTER 10.3.0.SW is n-1 (see COMPAT_GUIDS ref below.)
                 seen_group_guids.update([group["group_guid"] for group in d])
 
                 # DUMP GROUP->GROUP_MEMBERSHIP DATA
-                d = api_transformer.ts_group_membership(data=_, cluster=CLUSTER_UUID)
+                d = api_transformer.to_group_membership(data=_, cluster=CLUSTER_UUID)
                 temp.dump(models.GroupMembership.__tablename__, data=d)
 
             with tracker["TS_PRIVILEGE"]:
                 # TODO: ROLE->PRIVILEGE DATA.
                 # TODO: GROUP->ROLE DATA.
 
                 # DUMP GROUP->PRIVILEGE DATA
-                d = api_transformer.ts_group_privilege(data=_, cluster=CLUSTER_UUID)
+                d = api_transformer.to_group_privilege(data=_, cluster=CLUSTER_UUID)
                 temp.dump(models.GroupPrivilege.__tablename__, data=d)
 
-            if org["id"] == 0 and not primary_org_done:
+            if org["id"] == 0 or collect_info:
                 with tracker["TS_USER"]:
                     c = workflows.paginator(ts.api.users_search, record_size=5_000, timeout=60 * 15)
                     _ = utils.run_sync(c)
@@ -488,7 +497,7 @@ def metadata(
                     # DUMP USER->GROUP_MEMBERSHIP DATA
                     d = api_transformer.ts_group_membership(data=_, cluster=CLUSTER_UUID)
                     temp.dump(models.GroupMembership.__tablename__, data=d)
-                primary_org_done = True
+                collect_info = False
             elif org["id"] != 0:
                 log.info(f"Skipping USER data fetch for non-primary org (ID: {org['id']}) as it was already fetched.")
 

cs_tools/cli/tools/user-management/app.py

@@ -244,7 +244,7 @@ def delete(
         with tracker["DELETE"] as this_task:
             this_task.total = len(user_identifiers)
 
-            users_to_delete: set[_types.GUID] = {metadata_object["guid"] for metadata_object in user_identifiers}
+            users_to_delete: set[_types.GUID] = user_identifiers
             delete_attempts = collections.defaultdict(int)
 
             async def _delete_and_advance(guid: _types.GUID) -> None:
@@ -333,23 +333,27 @@ def sync(
             CLUSTER_UUID = ts.session_context.thoughtspot.cluster_id
 
             with tracker["TS_GROUP"]:
-                c = workflows.paginator(ts.api.groups_search, record_size=150_000, timeout=60 * 15)
-                _ = utils.run_sync(c)
+                c = ts.api.groups_search_v1()
+                r = utils.run_sync(c)
+                _ = r.json()
+
+                # c = workflows.paginator(ts.api.groups_search, record_size=150_000, timeout=60 * 15)
+                # _ = utils.run_sync(c)
 
                 # DUMP GROUP DATA
-                d = searchable.api_transformer.ts_group(data=_, cluster=CLUSTER_UUID)
+                d = searchable.api_transformer.to_group_v1(data=_, cluster=CLUSTER_UUID)
                 existing[searchable.models.Group.__tablename__].extend(d)
 
                 # DUMP GROUP->GROUP_MEMBERSHIP DATA
-                d = searchable.api_transformer.ts_group_membership(data=_, cluster=CLUSTER_UUID)
+                d = searchable.api_transformer.to_group_membership(data=_, cluster=CLUSTER_UUID)
                 existing[searchable.models.GroupMembership.__tablename__].extend(d)
 
             with tracker["TS_PRIVILEGE"]:
                 # TODO: ROLE->PRIVILEGE DATA.
                 # TODO: GROUP->ROLE DATA.
 
                 # DUMP GROUP->PRIVILEGE DATA
-                d = searchable.api_transformer.ts_group_privilege(data=_, cluster=CLUSTER_UUID)
+                d = searchable.api_transformer.to_group_privilege(data=_, cluster=CLUSTER_UUID)
                 existing[searchable.models.GroupPrivilege.__tablename__].extend(d)
 
             with tracker["TS_USER"]: