Added Directory API processor, Cloud Asset Inventory processor and (G…

…oogleCloudPlatform#692)

Cloud Ops Monitoring processor with corresponding examples.

tools/pubsub2inbox/README.md

@@ -32,12 +32,15 @@ Out of the box, you'll have the following functionality:
   - [Recommendations and Insights reports](examples/recommendations-example.yaml)
      - From [Recommender API](https://cloud.google.com/recommender/docs/overview).
      - Also see [example with attached spreadsheet](examples/recommendations-example-2.yaml).
-  - [Cloud Monitoring alerts](examples/monitoring-config.yaml)
+  - [Cloud Monitoring alerts](examples/monitoring-alert-config.yaml)
+  - [Cloud Monitoring metrics](examples/cai.yaml)
+  - [Cloud Asset Inventory search](examples/cai.yaml)
   - [Cloud Storage copier](examples/gcscopy-example.yaml)
      - Copies objects between two buckets, useful for backing up.
   - [Cloud Identity groups](examples/groups-example.yaml) ([other example](examples/groups-example-2.yaml))
      - Retrieves group and membership information from [Cloud Identity Groups API](https://cloud.google.com/identity/docs/apis)
      - Useful for example building membership review reports
+  - [Groups that allow external members](examples/external-groups-example.yaml) ([general example for Directory API](examples/directory-example.yaml))
   - Any JSON
     - [See the example of generic JSON processing](examples/generic-config.yaml)
 

tools/pubsub2inbox/examples/cai-example.yaml

@@ -0,0 +1,63 @@
+#   Copyright 2021 Google LLC
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+retryPeriod: 3 day ago
+
+# Example for using the Cloud Asset Inventory processor, which looks up 
+# all organizations service accounts and then queries their metrics to 
+# produce a report.
+#
+# The processor requires Cloud Asset Viewer permission on the correct
+# level.
+
+processors:
+  - cai
+  - monitoring
+
+cai:
+  parent: organizations/1234567890
+  # parent: projects/123456789
+  assetTypes:
+    - iam.googleapis.com/ServiceAccount
+    # - iam.googleapis.com/ServiceAccountKey
+
+  # See: https://cloud.google.com/asset-inventory/docs/reference/rest/v1/feeds#ContentType
+  contentType: "RESOURCE"
+
+  # readTime: "2021-09-13T14:14:23.045123456Z"
+
+monitoring:
+  timeSeries: "{{ assets['iam.googleapis.com/ServiceAccount']|json_encode }}"
+  # Specify your monitoring metric scope project below, this only works if every project
+  # is in its own metric scope.
+  #project: |
+  #  projects/{{ item.resource.data.projectId }}
+  key: |
+    {{ item.resource.data.email }}
+  query: |
+    fetch iam_service_account
+    | metric iam.googleapis.com/service_account/authn_events_count
+    | filter (resource.unique_id == '{{ item.resource.data.uniqueId }}')
+    | group_by 14d, [.sum]
+    | every 14d
+
+outputs:
+  - type: logger
+    level: debug
+    message: |
+      Service Account Summary (use in last 14 days)
+
+      {% for sa, ts in time_series.items() %}
+        - {{ sa }}: {% if ts.int64s|first|length > 0 %}{{ ts.int64s|first|first }} times{% else %}not used{% endif %} 
+      {% endfor %}

tools/pubsub2inbox/examples/directory-example.yaml

@@ -0,0 +1,54 @@
+#   Copyright 2021 Google LLC
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+retryPeriod: 3 day ago
+
+# This is an example for Directory API processor, which allows fetching of
+# groups, users, group memberships and group settings from the appropriate
+# APIs.
+
+processors:
+  - directory
+
+directory:
+  ### Fetching list of groups ###
+  api: groups
+  # Either use customer ID (directory ID) or domain:
+  # customerId: C00000000
+  # domain: example.com
+  # For query syntax, see:
+  # https://developers.google.com/admin-sdk/directory/v1/guides/search-groups
+  query: ""
+
+  ### Fetching group settings ###
+  # api: groupsettings
+  # groupUniqueIds: ["group@example.com"]
+
+  ### Fetching list of group members ###
+  # api: members
+  # groupKey: 04i7ojhp37zlnuw
+
+  ### Fetching list of users ###
+  # api: users
+  # (domain or customerId, query, etc)
+
+  # Specify the service account for getting the token (needs to have appropriate
+  # permissions in admin.google.com)
+  serviceAccountEmail: service-account@your-project.iam.gserviceaccount.com
+
+outputs:
+  - type: logger
+    level: debug
+    message: |
+      {{ results|json_encode() }}

tools/pubsub2inbox/examples/external-groups-example.yaml

@@ -0,0 +1,53 @@
+#   Copyright 2021 Google LLC
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+retryPeriod: 3 day ago
+
+# This example shows Directory API called twice, first to retrieve
+# all groups and then to retrieve group settings for those groups.
+# Then finally, all groups that allow external members are printed
+# out.
+#
+# You could also use Pub/Sub fanout to parallelize the processing
+# (see groups-example.yaml)
+# 
+processors:
+  - { processor: directory, config: get_groups }
+  - { processor: directory, config: get_group_settings }
+
+get_groups:
+  ### Fetching list of groups ###
+  api: groups
+  domain: example.com
+  query: ""
+  serviceAccountEmail: service-account@your-project.iam.gserviceaccount.com
+
+get_group_settings:
+  ### Fetching list of groups ###
+  api: groupsettings
+  domain: example.com
+  groupUniqueIds: |
+    {{ results|map(attribute='email')|list|json_encode }}
+  query: ""
+  serviceAccountEmail: service-account@your-project.iam.gserviceaccount.com
+
+outputs:
+  - type: logger
+    level: debug
+    message: |
+      The following groups allow external members:
+
+      {% for group in results|selectattr('allowExternalMembers', 'equalto', 'true') %}
+        - {{ group['email'] }} ({{ group['name'] }})
+      {% endfor %}

tools/pubsub2inbox/main.py

@@ -131,6 +131,12 @@ def process_message(config, data, event, context):
     jinja_environment = get_jinja_environment()
     if 'processors' in config:
         for processor in config['processors']:
+            config_key = None
+            if isinstance(processor, dict):
+                config_key = processor[
+                    'config'] if 'config' in processor else None
+                processor = processor['processor']
+
             logger.debug('Processing message using input processor: %s' %
                          processor)
             mod = __import__('processors.%s' % processor)
@@ -139,7 +145,8 @@ def process_message(config, data, event, context):
                                       '%sProcessor' % processor.capitalize())
             processor_instance = processor_class(config, jinja_environment,
                                                  data, event, context)
-            processor_variables = processor_instance.process()
+            processor_variables = processor_instance.process(
+                config_key=config_key)
             template_variables.update(processor_variables)
             jinja_environment.globals = {
                 **jinja_environment.globals,

tools/pubsub2inbox/processors/base.py

@@ -135,6 +135,12 @@ def _jinja_expand_bool(self, contents, _tpl='config'):
             return True
         return False
 
+    def _jinja_expand_string(self, contents, _tpl='config'):
+        var_template = self.jinja_environment.from_string(contents)
+        var_template.name = _tpl
+        val_str = var_template.render()
+        return val_str
+
     def _jinja_var_to_list(self, _var, _tpl='config'):
         if isinstance(_var, list):
             return _var
@@ -153,5 +159,5 @@ def _jinja_var_to_list(self, _var, _tpl='config'):
                 return list(map(lambda x: x.strip(), vals))
 
     @abc.abstractmethod
-    def process(self):
+    def process(self, config_key=None):
         pass

tools/pubsub2inbox/processors/bigquery.py

@@ -19,12 +19,14 @@
 
 class BigqueryProcessor(Processor):
 
-    def process(self):
-        if 'bigquery' not in self.config:
+    def process(self, config_key=None):
+        if config_key is None:
+            config_key = 'bigquery'
+        if config_key not in self.config:
             raise NotConfiguredException(
                 'No BigQuery configuration specified in config!')
 
-        bigquery_config = self.config['bigquery']
+        bigquery_config = self.config[config_key]
         if 'query' not in bigquery_config:
             raise NotConfiguredException(
                 'No BigQuery query specified in configuration!')

tools/pubsub2inbox/processors/cai.py

@@ -0,0 +1,61 @@
+#   Copyright 2021 Google LLC
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+from .base import Processor, NotConfiguredException
+from googleapiclient import discovery
+
+
+class CaiProcessor(Processor):
+
+    def process(self, config_key=None):
+        if config_key is None:
+            config_key = 'cai'
+        if config_key not in self.config:
+            raise NotConfiguredException('No settings configured!')
+
+        cai_config = self.config[config_key]
+        if 'parent' not in cai_config:
+            raise NotConfiguredException('No parent configured!')
+
+        cai_service = discovery.build('cloudasset', 'v1')
+
+        request_parameters = {}
+        request_parameters['parent'] = self._jinja_expand_string(
+            cai_config['parent'])
+
+        for k in ['readTime', 'pageSize', 'contentType']:
+            if k in cai_config:
+                request_parameters[k] = self._jinja_expand_string(cai_config[k])
+        if 'assetTypes' in cai_config:
+            request_parameters['assetTypes'] = self._jinja_var_to_list(
+                cai_config['assetTypes'])
+
+        assets = {}
+        page_token = None
+        while True:
+            if page_token is not None:
+                request_parameters['pageToken'] = page_token
+            request = cai_service.assets().list(**request_parameters)
+            response = request.execute()
+            if 'assets' in response:
+                for asset in response['assets']:
+                    if asset['assetType'] not in assets:
+                        assets[asset['assetType']] = []
+                    assets[asset['assetType']].append(asset)
+
+            if 'nextPageToken' in response:
+                page_token = response['nextPageToken']
+            else:
+                break
+
+        return {'assets': assets}