Added queries to limit the scan scope via the --scope input param

src/n0s1/__init__.py

@@ -1 +1 @@
-__version__ = "1.0.26"
+__version__ = "1.0.27"

src/n0s1/controllers/confluence_controller.py

@@ -214,6 +214,32 @@ def get_mapping(self, levels=-1, limit=None):
     def get_data(self, include_coments=False, limit=None):
         if not self._client:
             return {}
+
+        pages = []
+        using_cql = False
+        cql = self.get_query_from_scope()
+        if cql:
+            try:
+                res = self._client.cql(cql, limit=limit)
+                results = res.get("results", [])
+                for r in results:
+                    content_type = r.get("content", {}).get("type", None)
+                    if content_type and content_type.lower() == "page".lower():
+                        pages.append(r.get("content", {}))
+                if len(pages) > 0:
+                    using_cql = True
+                    yield from self.process_pages(include_coments, limit, pages)
+                else:
+                    message = f"No pages found for [cql:{cql}]. Scan will not be scoped."
+                    self.log_message(message, logging.WARNING)
+                    self._scan_scope = None
+            except Exception as e:
+                message = str(e) + f" cql({cql}, limit={limit})"
+                self.log_message(message, logging.WARNING)
+
+        if using_cql:
+            return
+
         for spaces in self._get_workspaces(limit):
             for s in spaces:
                 key = s
@@ -222,49 +248,54 @@ def get_data(self, include_coments=False, limit=None):
                 self.log_message(f"Scanning Confluence space: [{key}]...")
                 if len(key) > 0:
                     for pages in self._get_pages(key, limit):
-                        for p in pages:
-                            comments = []
-                            title = p.get("title", "")
-                            page_id = p.get("id", "")
-                            try:
-                                self.connect()
-                                body = self._client.get_page_by_id(page_id, expand="body.storage")
-                            except Exception as e:
-                                message = str(e) + f" get_page_by_id({page_id})"
-                                self.log_message(message, logging.WARNING)
-                                body = {}
-                                time.sleep(1)
-                                continue
+                        yield from self.process_pages(include_coments, limit, pages)
+
+    def process_pages(self, include_coments, limit, pages):
+        for p in pages:
+            comments = []
+            title = p.get("title", "")
+            page_id = p.get("id", "")
+            try:
+                self.connect()
+                body = self._client.get_page_by_id(page_id, expand="body.storage")
+            except Exception as e:
+                message = str(e) + f" get_page_by_id({page_id})"
+                self.log_message(message, logging.WARNING)
+                body = {}
+                time.sleep(1)
+                continue
 
-                            description = body.get("body", {}).get("storage", {}).get("value", "")
-                            url = body.get("_links", {}).get("base", "") + p.get("_links", {}).get("webui", "")
-                            if len(page_id) > 0 and include_coments:
-                                if not limit or limit < 0:
-                                    limit = 50
-                                comments_start = 0
-                                comments_finished = False
-                                while not comments_finished:
-                                    try:
-                                        self.connect()
-                                        comments_response = self._client.get_page_comments(page_id, expand="body.storage", start=comments_start, limit=limit)
-                                        comments_result = comments_response.get("results", [])
-                                    except Exception as e:
-                                        message = str(e) + f" get_page_comments({page_id}, expand=\"body.storage\", start={comments_start}, limit={limit})"
-                                        self.log_message(message, logging.WARNING)
-                                        comments_result = [{}]
-                                        time.sleep(1)
-                                        continue
-                                    comments_start += limit
+            description = body.get("body", {}).get("storage", {}).get("value", "")
+            url = body.get("_links", {}).get("base", "") + p.get("_links", {}).get("webui", "")
+            if len(page_id) > 0 and include_coments:
+                if not limit or limit < 0:
+                    limit = 50
+                comments_start = 0
+                comments_finished = False
+                while not comments_finished:
+                    try:
+                        self.connect()
+                        comments_response = self._client.get_page_comments(page_id, expand="body.storage",
+                                                                           start=comments_start, limit=limit)
+                        comments_result = comments_response.get("results", [])
+                    except Exception as e:
+                        message = str(
+                            e) + f" get_page_comments({page_id}, expand=\"body.storage\", start={comments_start}, limit={limit})"
+                        self.log_message(message, logging.WARNING)
+                        comments_result = [{}]
+                        time.sleep(1)
+                        continue
+                    comments_start += limit
 
-                                    for c in comments_result:
-                                        comment = c.get("body", {}).get("storage", {}).get("value", "")
-                                        comments.append(comment)
+                    for c in comments_result:
+                        comment = c.get("body", {}).get("storage", {}).get("value", "")
+                        comments.append(comment)
 
-                                    if len(comments_result) <= 0:
-                                        comments_finished = True
+                    if len(comments_result) <= 0:
+                        comments_finished = True
 
-                            ticket = self.pack_data(title, description, comments, url, page_id)
-                            yield ticket
+            ticket = self.pack_data(title, description, comments, url, page_id)
+            yield ticket
 
     def post_comment(self, issue, comment):
         if not self._client:

src/n0s1/controllers/github_controller.py

@@ -180,7 +180,12 @@ def get_data(self, include_comments=False, limit=None):
         if not self._client:
             return {}
 
-        repos, owner = self._get_repos()
+        repos = None
+        q = self.get_query_from_scope()
+        if q:
+            repos = self._client.search_repositories(query=q)
+        if not repos:
+            repos, owner = self._get_repos()
         if repos:
             for repo in repos:
                 if isinstance(repo, str):

src/n0s1/controllers/hollow_controller.py

@@ -76,3 +76,15 @@ def pack_data(self, title, description, comments, url, ticket_key):
             "issue_id": ticket_key
         }
         return ticket_data
+
+    def get_query_from_scope(self):
+        query = None
+        if self._scan_scope:
+            query = self._scan_scope.get("query", None)
+            if not query:
+                query = self._scan_scope.get("search", None)
+            if not query:
+                query = self._scan_scope.get("jql", None)
+            if not query:
+                query = self._scan_scope.get("cql", None)
+        return query

src/n0s1/controllers/jira_controller.py

@@ -152,7 +152,18 @@ def get_data(self, include_coments=False, limit=None):
             return {}
         try:
             self.connect()
-            projects = self._get_projects(limit)
+            using_jql = False
+            jql = self.get_query_from_scope()
+            if jql:
+                issues = self._client.search_issues(jql)
+                for issue in issues:
+                    ticket = self._extract_ticket(include_coments, issue)
+                    using_jql = True
+                    yield ticket
+            if using_jql:
+                projects = []
+            else:
+                projects = self._get_projects(limit)
         except Exception as e:
             message = str(e) + f" client.projects()"
             self.log_message(message, logging.WARNING)
@@ -162,24 +173,27 @@ def get_data(self, include_coments=False, limit=None):
             self.log_message(f"Scanning Jira project: [{key}]...")
             for issues in self._get_issues(key, limit):
                 for issue in issues:
-                    url = issue.self.split('/rest/api')[0] + "/browse/" + issue.key;
-                    title = issue.fields.summary
-                    description = issue.fields.description
-                    comments = []
-                    if include_coments:
-                        try:
-                            self.connect()
-                            issue_comments = self._client.comments(issue.id)
-                            comments.extend(c.body for c in issue_comments)
-                        except Exception as e:
-                            message = str(e) + f" client.comments({issue.id})"
-                            self.log_message(message, logging.WARNING)
-                            comments = []
-                            time.sleep(1)
-
-                    ticket = self.pack_data(title, description, comments, url, issue.key)
+                    ticket = self._extract_ticket(include_coments, issue)
                     yield ticket
 
+    def _extract_ticket(self, include_coments, issue):
+        url = issue.self.split('/rest/api')[0] + "/browse/" + issue.key;
+        title = issue.fields.summary
+        description = issue.fields.description
+        comments = []
+        if include_coments:
+            try:
+                self.connect()
+                issue_comments = self._client.comments(issue.id)
+                comments.extend(c.body for c in issue_comments)
+            except Exception as e:
+                message = str(e) + f" client.comments({issue.id})"
+                self.log_message(message, logging.WARNING)
+                comments = []
+                time.sleep(1)
+        ticket = self.pack_data(title, description, comments, url, issue.key)
+        return ticket
+
     def post_comment(self, issue, comment):
         if not self._client:
             return False

src/n0s1/controllers/slack_controller.py

@@ -33,6 +33,19 @@ def is_connected(self):
         return True
 
     def get_data(self, include_coments=False, limit=None):
+
+        using_scan_scope = False
+        query = self.get_query_from_scope()
+        if query:
+            messages = self.run_slack_query(query)
+            for m in messages:
+                if len(m) > 0:
+                    using_scan_scope = True
+                    yield from self._extract_ticket(m)
+
+        if using_scan_scope:
+            return
+
         max_day_range = 365 * 100
         range_days = 1
         now = datetime.datetime.now()
@@ -56,12 +69,7 @@ def get_data(self, include_coments=False, limit=None):
                     range_days = range_days * 2
                 else:
                     range_days = 1
-                    for item in m:
-                        message = item.get("text", "")
-                        iid = item.get("iid", "")
-                        url = item.get("permalink", "")
-                        ticket = self.pack_data(message, item, url, iid)
-                        yield ticket
+                    yield from self._extract_ticket(m)
 
             end_day = start_day + datetime.timedelta(days=1)
             start_day = start_day - datetime.timedelta(days=range_days)
@@ -70,6 +78,14 @@ def get_data(self, include_coments=False, limit=None):
             query = f"after:{start_day_str} before:{end_day_str}"
             days_counter += range_days
 
+    def _extract_ticket(self, message):
+        for item in message:
+            message = item.get("text", "")
+            iid = item.get("iid", "")
+            url = item.get("permalink", "")
+            ticket = self.pack_data(message, item, url, iid)
+            yield ticket
+
     def post_comment(self, issue, comment):
         from slack_sdk.errors import SlackApiError
         try:

src/n0s1/controllers/zendesk_controller.py

@@ -45,8 +45,18 @@ def get_data(self, include_comments=False, limit=None):
 
         try:
             server = self._config.get("server", "")
-            # Fetch all tickets (paginated)
-            tickets = self._client.tickets()
+
+            using_scan_scope = False
+            query = self.get_query_from_scope()
+            if query:
+                tickets = self._client.search(query=query)
+                if len(tickets) > 0:
+                    using_scan_scope = True
+
+            if not using_scan_scope:
+                # Fetch all tickets (paginated)
+                tickets = self._client.tickets()
+
             for ticket in tickets:
                 self.log_message(f"Scanning Zendesk Ticket ID: {ticket.id}, Subject: {ticket.subject}, Status: {ticket.status}, Created: {ticket.created_at}")
                 comments = []

src/n0s1/n0s1.py

@@ -204,7 +204,7 @@ def init_argparse() -> argparse.ArgumentParser:
         nargs="?",
         default="Disabled",
         type=str,
-        help="Define a chunk of the map file to be scanned. Ex: 3/4 (will scan the third quarter of the map)."
+        help="Define a search query Ex: \"search:org:spark1security action in:name\" for GitHub or \"jql:project != IT\" for Jira. If using with --map-file, it defines a chunk of the map file to be scanned. Ex: 3/4 (will scan the third quarter of the map)."
     )
     subparsers = parser.add_subparsers(
         help="Subcommands", dest="command", metavar="COMMAND"
@@ -416,6 +416,11 @@ def _safe_re_search(regex_str, text):
 def match_regex(regex_config, text):
     for c in regex_config["rules"]:
         regex_str = c["regex"]
+        modifiers = ["(?i)", "(?m)", "(?s)", "(?x)", "(?g)", "(?u)", "(?A)", "(?L)", "(?U)", ]
+        for modifier in modifiers:
+            if regex_str.find(modifier) > 0:
+                regex_str = regex_str.replace(modifier, "")
+                regex_str = modifier + regex_str
         if m := _safe_re_search(regex_str, text):
             begin = m.regs[0][0]
             end = m.regs[0][1]
@@ -599,7 +604,10 @@ def main(callback=None):
 
     scope_config = get_scope_config(args)
     if scope_config:
-        log_message(f"Running scoped scan using map file [{args.map_file}]. Scan scope:", level=logging.INFO)
+        if args.map_file:
+            log_message(f"Running scoped scan using map file [{args.map_file}]. Scan scope:", level=logging.INFO)
+        else:
+            log_message(f"Running scoped scan using search query:", level=logging.INFO)
         pprint.pprint(scope_config)
 
     datetime_now_obj = datetime.now(timezone.utc)
@@ -819,6 +827,15 @@ def main(callback=None):
 
 def get_scope_config(args):
     scope_config = None
+    if args.scope:
+        scope_terms = ["jql", "cql", "search", "query"]
+        for t in scope_terms:
+            query_index = args.scope.lower().replace(" ", "").find(f"{t}:".lower())
+            if query_index == 0:
+                query = args.scope[len(t)+1:]
+                scope_config = {t: query}
+                return scope_config
+
     if args.map and args.map.lower() != "Disabled".lower():
         # New mapping. Skipp loading old mapped scope
         return scope_config