Merge pull request #235 from TS-sumeet-jain/dev

Fix for Archiver, Searchable & User management

Author: billdback-ts

cs_tools/__project__.py

@@ -1,4 +1,4 @@
-__version__ = "1.6.3"
+__version__ = "1.6.4"
 __docs__ = "https://thoughtspot.github.io/cs_tools/"
 __repo__ = "https://github.com/thoughtspot/cs_tools"
 __help__ = f"{__repo__}/discussions/"

cs_tools/api/client.py

@@ -436,6 +436,13 @@ def groups_search_v1(self, **options: Any) -> Awaitable[httpx.Response]:  # noqa
         """Get a list of ThoughtSpot groups with v1 endpoint."""
         return self.get("callosum/v1/tspublic/v1/group")
 
+    @pydantic.validate_call(validate_return=True, config=validators.METHOD_CONFIG)
+    @_transport.CachePolicy.mark_cacheable
+    def group_list_users(self, group_guid: _types.ObjectIdentifier, **options: Any) -> Awaitable[httpx.Response]:  # noqa: ARG002
+        """Get a list of ThoughtSpot users in a group with v1 endpoint."""
+        r = self.get(f"callosum/v1/tspublic/v1/group/{group_guid}/users")
+        return r
+
     @pydantic.validate_call(validate_return=True, config=validators.METHOD_CONFIG)
     def groups_create(self, **options: Any) -> Awaitable[httpx.Response]:
         """Create a ThoughtSpot group."""

cs_tools/cli/tools/archiver/app.py

@@ -41,6 +41,19 @@ def _tick_tock(task: px.WorkTask) -> None:
         task.advance(step=1)
 
 
+def get_group_users(guids: list[_types.ObjectIdentifier], users_profiles: list[dict]):
+    users = []
+    for guid in guids:
+        for user in users_profiles:
+            if guid in user["assignedGroups"]:
+                users.append(user["header"])
+
+            if guid in user["inheritedGroups"]:
+                users.append(user["header"])
+
+    return users
+
+
 app = AsyncTyper(
     help="""
     Manage stale answers and liveboards within your platform.
@@ -156,7 +169,7 @@ def identify(
 
             SEARCH_TOKENS = (
                 # SELECT TS: BI ACTIVITY ON ANY ACTIVITY WITH A QUERY
-                "[query text] != '{null}' "
+                # "[query text] != '{null}' "
                 # FILTER OUT AD-HOC SEARCH
                 "[user action] != [user action].answer_unsaved "
                 # FILTER OUT POTENTIAL DATA QUALITY ISSUES ? (just here for safety~)
@@ -187,22 +200,67 @@ def identify(
             if only_groups or ignore_groups:
                 c = ts.api.groups_search_v1()
                 r = utils.run_sync(c)
-                _ = r.json()
-
-                # c = workflows.paginator(ts.api.groups_search, record_size=150_000, timeout=60 * 15)
-                # d = utils.run_sync(c)
+                d = r.json()
 
+                # Inline with V1 Group API.
                 all_groups = [
                     {
-                        "guid": group["id"],
-                        "name": group["name"],
-                        "users": group["users"],
+                        "guid": group["header"]["id"],
+                        "name": group["header"]["name"],
                     }
                     for group in d
                 ]
 
-                only_groups = [group["guid"] for group in all_groups if group["name"] in (only_groups or [])]  # type: ignore[assignment]
-                ignore_groups = [group["guid"] for group in all_groups if group["name"] in (ignore_groups or [])]  # type: ignore[assignment]
+                # c = workflows.paginator(ts.api.groups_search, record_size=150_000, timeout=60 * 15)
+                # d = utils.run_sync(c)
+
+                # all_groups = [
+                #     {
+                #         "guid": group["id"],
+                #         "name": group["name"],
+                #         "users": group["users"],
+                #     }
+                #     for group in d
+                # ]
+
+                only_groups_lst = [group["guid"] for group in all_groups if group["name"] in (only_groups or [])]  # type: ignore[assignment]
+                ignore_groups_lst = [group["guid"] for group in all_groups if group["name"] in (ignore_groups or [])]  # type: ignore[assignment]
+
+                if only_groups is not None:
+                    users_profiles: list = []
+                    for guid in only_groups_lst:
+                        c = ts.api.group_list_users(group_guid=guid)
+                        r = r = utils.run_sync(c)
+                        users = r.json()
+                        if not users:
+                            # Exception needs to be redone
+                            info = {
+                                "reason": "Group names are case sensitive. "
+                                + "You can find a group's 'Group Name' in the Admin panel.",
+                                "mitigation": "Verify the name and try again.",
+                                "type": "Group",
+                            }
+                            raise Exception(str(info)) from None
+                        users_profiles.extend(users)
+                    users_only = [user["id"] for user in get_group_users(only_groups_lst, users_profiles)]
+
+                if ignore_groups is not None:
+                    users_profiles: list = []
+                    for guid in ignore_groups_lst:
+                        c = ts.api.group_list_users(group_guid=guid)
+                        r = utils.run_sync(c)
+                        users = r.json()
+                        if not users:
+                            info = {
+                                "reason": "Group names are case sensitive. "
+                                + "You can find a group's 'Group Name' in the Admin panel.",
+                                "mitigation": "Verify the name and try again.",
+                                "type": "Group",
+                            }
+                            raise Exception(str(info)) from None
+                        users_profiles.extend(users)
+
+                    users_ignore = [user["id"] for user in get_group_users(ignore_groups_lst, users_profiles)]
 
             if ignore_tags:
                 c = workflows.metadata.fetch_all(metadata_types=["TAG"], http=ts.api)
@@ -259,18 +317,18 @@ def identify(
                 # CHECK: THE AUTHOR IS A MEMBER OF GROUPS WHOSE CONTENT SHOULD NEEDS TO INCLUDED.
                 if only_groups is not None:
                     assert isinstance(only_groups, list), "Only Groups wasn't properly transformed to an array<GUID>."
-                    checks.append(metadata_object["author_guid"] in only_groups)
+                    checks.append(metadata_object["author_guid"] in users_only)
 
                 # CHECK: THE AUTHOR IS NOT A MEMBER OF GROUPS WHOSE CONTENT SHOULD BE IGNORED.
                 if ignore_groups is not None:
                     assert isinstance(
                         ignore_groups, list
                     ), "Ignore Groups wasn't properly transformed to an array<GUID>."
-                    checks.append(metadata_object["author_guid"] not in ignore_groups)
+                    checks.append(metadata_object["author_guid"] not in users_ignore)
 
                 if ignore_tags is not None:
                     assert isinstance(ignore_tags, list), "Ignore Tags wasn't properly transformed to an array<GUID>."
-                    checks.append(any(t["id"] not in ignore_tags for t in metadata_object["tags"]))
+                    checks.append(not any(t["id"] in ignore_tags for t in metadata_object["tags"]))
 
                 if all(checks):
                     filtered.append(metadata_object)

cs_tools/cli/tools/searchable/app.py

@@ -428,7 +428,11 @@ def metadata(
         tracker["ORGS_COUNT"].start()
 
         # LOOP THROUGH EACH ORG COLLECTING DATA
-        primary_org_done = False
+        if org_override is not None:
+            collect_info = True
+        else:
+            collect_info = False
+
         for org in orgs:
             tracker.title = f"Fetching Data in [fg-secondary]{org['name']}[/] (Org {org['id']})"
             seen_guids: dict[_types.APIObjectType, set[_types.GUID]] = collections.defaultdict(set)
@@ -477,7 +481,7 @@ def metadata(
                 d = api_transformer.to_group_privilege(data=_, cluster=CLUSTER_UUID)
                 temp.dump(models.GroupPrivilege.__tablename__, data=d)
 
-            if org["id"] == 0 and not primary_org_done:
+            if org["id"] == 0 or collect_info:
                 with tracker["TS_USER"]:
                     c = workflows.paginator(ts.api.users_search, record_size=5_000, timeout=60 * 15)
                     _ = utils.run_sync(c)
@@ -493,7 +497,7 @@ def metadata(
                     # DUMP USER->GROUP_MEMBERSHIP DATA
                     d = api_transformer.ts_group_membership(data=_, cluster=CLUSTER_UUID)
                     temp.dump(models.GroupMembership.__tablename__, data=d)
-                primary_org_done = True
+                collect_info = False
             elif org["id"] != 0:
                 log.info(f"Skipping USER data fetch for non-primary org (ID: {org['id']}) as it was already fetched.")
 

cs_tools/cli/tools/user-management/app.py

@@ -244,7 +244,7 @@ def delete(
         with tracker["DELETE"] as this_task:
             this_task.total = len(user_identifiers)
 
-            users_to_delete: set[_types.GUID] = {metadata_object["guid"] for metadata_object in user_identifiers}
+            users_to_delete: set[_types.GUID] = user_identifiers
             delete_attempts = collections.defaultdict(int)
 
             async def _delete_and_advance(guid: _types.GUID) -> None:

pyproject.toml

@@ -35,13 +35,14 @@ classifiers = [
 dependencies = [
     # dependencies here are listed for cs_tools the library  (ie. import cs_tools)
     "aiosqlite",
+    "click == 8.1.7",
     "thoughtspot-tml",
     "awesomeversion",
     "httpx >= 0.27.0",
     "pydantic >= 2.6.4",
     "pydantic-settings",
     "email-validator",
-    "rich >= 13.7.1",
+    "rich == 13.7.1",
     "sqlmodel >= 0.0.16",
     "tenacity",
     "toml",