-
Notifications
You must be signed in to change notification settings - Fork 13
/
collector.py
117 lines (102 loc) · 5.04 KB
/
collector.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
import logging
import sys
import csv
from gcp_iam_iterator import GcpIamIterator
logging.basicConfig(
format='%(asctime)s %(levelname)-5s %(filename)-12s %(message)s',
level=logging.DEBUG, stream=sys.stdout)
def dump_projects(iam_iterator):
with open('projects.csv', 'wb') as csvfile:
writer = csv.writer(csvfile)
for counter, project in enumerate(iam_iterator.list_projects()):
project_id = project['projectId']
logging.info("parsing project [{0}] projectId: {1}"
.format(counter, project_id))
writer.writerow([project_id, project['projectNumber'],
project['createTime'], project['name']])
def dump_projects_iam(iam_iterator):
with open('projects_iam.csv', 'wb') as csvfile:
writer = csv.writer(csvfile)
for counter, project in enumerate(iam_iterator.list_projects()):
project_id = project['projectId']
logging.info("parsing project [{0}] projectId: {1}"
.format(counter, project_id))
for binding in iam_iterator.list_project_iam(project_id):
role = binding['role']
for member in binding['members']:
member_type = member.split(":")[0]
member_name = member.split(":")[1]
writer.writerow(
[project_id, role, member_type, member_name])
def dump_service_accounts(iam_iterator):
with open('serviceAccounts.csv', 'wb') as csvfile:
writer = csv.writer(csvfile)
for counter, project in enumerate(iam_iterator.list_projects()):
project_id = project['projectId']
logging.info("parsing project [{0}] projectId: {1}"
.format(counter, project_id))
for account in iam_iterator.list_service_accounts(project_id):
email = account['email']
sa_key_exists = False
for sa_key in iam_iterator.list_service_account_keys(email):
sa_key_exists = True
writer.writerow(
[project_id, email, account.get('displayName', ''),
sa_key['validAfterTime'],
sa_key['validBeforeTime'],
sa_key['name'].split('/')[5]])
if not sa_key_exists:
writer.writerow(
[project_id, email, account.get('displayName', ''), '',
'', ''])
def dump_datasets_iam(iam_iterator):
with open('datasets_iam.csv', 'wb') as csvfile:
writer = csv.writer(csvfile)
for counter, project in enumerate(iam_iterator.list_projects()):
project_id = project['projectId']
logging.info("parsing project [{0}] projectId: {1}"
.format(counter, project_id))
for dataset in iam_iterator.list_datasets(project_id):
dataset_id = dataset['datasetReference']['datasetId']
for access in iam_iterator.list_dataset_access(
project_id=project_id, dataset_id=dataset_id):
if 'role' not in access:
continue
role = access['role']
iam_type = 'userByEmail' if 'userByEmail' in access \
else 'groupByEmail' if 'groupByEmail' in access \
else 'specialGroup' if 'specialGroup' in access \
else 'None'
member = access[iam_type]
writer.writerow(
[project_id, dataset_id, role, iam_type, member])
def dump_buckets_iam(iam_iterator):
with open('buckets_iam.csv', 'wb') as csvfile:
writer = csv.writer(csvfile)
for counter, project in enumerate(iam_iterator.list_projects()):
project_id = project['projectId']
logging.info("parsing project [{0}] projectId: {1}"
.format(counter, project_id))
for bucket in iam_iterator.list_buckets(project_id):
bucket_id = bucket['id']
for access in iam_iterator.list_bucket_access(
bucket_id=bucket_id):
role = access['role']
entity = access['entity']
if 'projectTeam' in access:
member_type = access['projectTeam']['team']
else:
member_type = entity.split('-')[0]
member = entity.split('-', 1)[
1] if '-' in entity else entity
writer.writerow(
[project_id, project['projectNumber'], bucket_id, role,
member_type, member,
entity])
if __name__ == '__main__':
iam_iterator = GcpIamIterator(use_cache=False)
dump_projects(iam_iterator)
dump_projects_iam(iam_iterator)
dump_service_accounts(iam_iterator)
dump_datasets_iam(iam_iterator)
dump_buckets_iam(iam_iterator)