Skip to content

Commit db7635b

Browse files
authored
Merge pull request #22 from mwaldronii/updating-prism-table
Prism API V2 Updates for Tables
2 parents 15123e7 + 42b7a4f commit db7635b

File tree

3 files changed

+78
-114
lines changed

3 files changed

+78
-114
lines changed

.github/workflows/pythonpackage.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ jobs:
2525
- name: Format with Black
2626
run: |
2727
pip install black
28-
black . --check
28+
black . --check --verbose --line-length=120
2929
- name: Lint with flake8
3030
run: |
3131
pip install flake8

README.md

Lines changed: 19 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
# Prism-Python
44

5-
Python client library and command line interface (CLI) for interacting with Workday’s Prism API.
5+
Python client library for interacting with Workday’s Prism API V2.
66

77
## Install
88
You may install the latest version by cloning this repository from GitHub
@@ -43,60 +43,6 @@ export prism_client_secret=<INSERT PRISM CLIENT SECRET HERE>
4343
export prism_refresh_token=<INSERT PRISM REFRESH TOKEN HERE>
4444
```
4545

46-
## Example: Create a new dataset with Prism API Version 1
47-
48-
### Using the CLI
49-
50-
```bash
51-
# view the help page
52-
prism --help
53-
54-
# list all datasets of type API
55-
prism list
56-
57-
# upload a gzip CSV file
58-
prism upload my_new_dataset /path/to/schema.json /path/to/file.csv.gz
59-
```
60-
61-
### Using as a Python Package
62-
63-
```python
64-
import os
65-
import prism
66-
67-
# initialize the prism class with your credentials
68-
p = prism.Prism(
69-
os.getenv("workday_base_url"),
70-
os.getenv("workday_tenant_name"),
71-
os.getenv("prism_client_id"),
72-
os.getenv("prism_client_secret"),
73-
os.getenv("prism_refresh_token"),
74-
version="v1"
75-
)
76-
77-
# create the bearer token
78-
p.create_bearer_token()
79-
80-
# create an empty API dataset
81-
dataset = p.create_dataset("my_new_dataset")
82-
83-
# read in your dataset schema
84-
schema = prism.load_schema("/path/to/schema.json")
85-
86-
# create a new bucket to hold your file
87-
bucket = p.create_bucket(schema, dataset["id"])
88-
89-
# add your file the bucket you just created
90-
p.upload_file_to_bucket(bucket["id"], "/path/to/file.csv.gz")
91-
92-
# complete the bucket and upload your file
93-
p.complete_bucket(bucket["id"])
94-
95-
# check the status of the dataset you just created
96-
status = p.list_dataset(dataset["id"])
97-
print(status)
98-
```
99-
10046
## Example: Create a new table with Prism API Version 2
10147

10248
```python
@@ -120,29 +66,32 @@ p.create_bearer_token()
12066
schema = prism.load_schema("/path/to/schema.json")
12167

12268
# create an empty API table with your schema
123-
table = p.create_dataset('my_new_table', schema=schema['fields'])
69+
table = p.create_table('my_new_table', schema=schema['fields'])
12470

12571
# get the details about the newly created table
126-
details = p.describe_dataset(table['id'])
72+
details = p.describe_table(table['id'])
12773

12874
# convert the details to a bucket schema
12975
bucket_schema = p.convert_describe_schema_to_bucket_schema(details)
13076

13177
# create a new bucket to hold your file
132-
bucket = p.create_bucket(bucket_schema, table['id'], operation="Replace")
78+
bucket = p.create_bucket(bucket_schema, table['id'], operation="TruncateandInsert")
13379

13480
# add your file the bucket you just created
13581
p.upload_file_to_bucket(bucket["id"], "/path/to/file.csv.gz")
13682

13783
# complete the bucket and upload your file
13884
p.complete_bucket(bucket["id"])
13985

140-
# check the status of the table you just created
141-
status = p.list_dataset(table['id'])
142-
print(status)
86+
# check the status of the bucket you just completed
87+
status = p.list_bucket(bucket["id"])
88+
print(status.get('errorMessage'))
14389
```
14490

145-
## Example: Append data to an existing table with Prism API Version 2
91+
## Example: Manage data in an existing table with Prism API Version 2
92+
Table Operations Available - “TruncateandInsert”, “Insert”, “Update”, “Upsert”, “Delete”.
93+
94+
To use the Update/Upsert/Delete operations you must specify an external id field within your table schema.
14695

14796
```python
14897
import os
@@ -162,30 +111,30 @@ p = prism.Prism(
162111
p.create_bearer_token()
163112

164113
# look through all of the existing tables to find the table you intend to append
165-
all_datasets = p.list_dataset()
166-
for table in all_datasets['data']:
114+
all_tables = p.list_table()
115+
for table in all_tables['data']:
167116
if table['name'] == "my_new_table":
168117
print(table)
169118
break
170119

171120
# get the details about the newly created table
172-
details = p.describe_dataset(table['id'])
121+
details = p.describe_table(table['id'])
173122

174123
# convert the details to a bucket schema
175124
bucket_schema = p.convert_describe_schema_to_bucket_schema(details)
176125

177126
# create a new bucket to hold your file
178-
bucket = p.create_bucket(bucket_schema, table['id'], operation="Append")
127+
bucket = p.create_bucket(bucket_schema, table['id'], operation="Insert")
179128

180-
# add your file the bucket you just created
129+
# add your file to the bucket you just created
181130
p.upload_file_to_bucket(bucket["id"], "/path/to/file.csv.gz")
182131

183132
# complete the bucket and upload your file
184133
p.complete_bucket(bucket["id"])
185134

186-
# check the status of the table you just created
187-
status = p.list_dataset(table['id'])
188-
print(status)
135+
# check the status of the bucket you just completed
136+
status = p.list_bucket(bucket["id"])
137+
print(status.get('errorMessage'))
189138
```
190139

191140
## Bugs

prism/prism.py

Lines changed: 58 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ def load_schema(filename):
3030
Returns
3131
-------
3232
schema : dict
33-
A dictionary containing the schema for your dataset.
33+
A dictionary containing the schema for your table.
3434
3535
"""
3636
with open(filename) as f:
@@ -63,8 +63,8 @@ class Prism:
6363
The version of the Prism API to use
6464
"""
6565

66-
def __init__(self, base_url, tenant_name, client_id, client_secret, refresh_token, version="v1"):
67-
"""Init the Prism class with required attribues."""
66+
def __init__(self, base_url, tenant_name, client_id, client_secret, refresh_token, version="v2"):
67+
"""Init the Prism class with required attributes."""
6868
self.base_url = base_url
6969
self.tenant_name = tenant_name
7070
self.client_id = client_id
@@ -82,7 +82,6 @@ def create_bearer_token(self):
8282
8383
Parameters
8484
----------
85-
None
8685
8786
Returns
8887
-------
@@ -107,13 +106,13 @@ def create_bearer_token(self):
107106
else:
108107
logging.warning("HTTP Error {}".format(r.status_code))
109108

110-
def create_dataset(self, dataset_name, schema=None):
111-
"""Create an empty dataset of type "API".
109+
def create_table(self, table_name, schema=None):
110+
"""Create an empty table of type "API".
112111
113112
Parameters
114113
----------
115-
dataset_name : str
116-
The dataset name. The name must be unique and conform to the name
114+
table_name : str
115+
The table name. The name must be unique and conform to the name
117116
validation rules.
118117
119118
schema : list
@@ -122,7 +121,7 @@ def create_dataset(self, dataset_name, schema=None):
122121
Returns
123122
-------
124123
If the request is successful, a dictionary containing information about
125-
the new dataset is returned.
124+
the new table is returned.
126125
127126
"""
128127
url = self.prism_endpoint + "/datasets"
@@ -132,35 +131,39 @@ def create_dataset(self, dataset_name, schema=None):
132131
"Content-Type": "application/json",
133132
}
134133

135-
data = {"name": dataset_name}
134+
data = {"name": table_name}
136135

137136
if schema is not None:
138137
data["fields"] = schema
139138

140139
r = requests.post(url, headers=headers, data=json.dumps(data))
141140

142141
if r.status_code == 201:
143-
logging.info("Successfully created an empty API dataset")
142+
logging.info("Successfully created an empty API table")
144143
return r.json()
145144
elif r.status_code == 400:
146145
logging.warning(r.json()["errors"][0]["error"])
147146
else:
148147
logging.warning("HTTP Error {}".format(r.status_code))
149148

150-
def create_bucket(self, schema, dataset_id, operation="Replace"):
149+
def create_bucket(self, schema, table_id, operation="TruncateandInsert"):
151150
"""Create a temporary bucket to upload files.
152151
153152
Parameters
154153
----------
155154
schema : dict
156-
A dictionary containing the schema for your dataset.
155+
A dictionary containing the schema for your table.
157156
158-
dataset_id : str
159-
The ID of the dataset that this bucket is to be associated with.
157+
table_id : str
158+
The ID of the table that this bucket is to be associated with.
160159
161160
operation : str
162-
If not specified, defaults to "Replace" operation
163-
Optional values - "Replace" or "Append"
161+
Required, defaults to "TruncateandInsert" operation
162+
Additional Operations - “Insert”, “Update”, “Upsert”, “Delete”
163+
When you use Update/Upsert/Delete operation you must specify which field to use
164+
as the matching key by setting the ‘useAsOperationKey’ attribute on that field as True.
165+
Only fields marked as ExternalID or WPA_RowID or WPA_LoadId on Table schema can be used
166+
as operation keys during loads into the table.
164167
165168
Returns
166169
-------
@@ -178,7 +181,7 @@ def create_bucket(self, schema, dataset_id, operation="Replace"):
178181
data = {
179182
"name": "prism_python_wbucket_" + str(random.randint(1000000, 9999999)),
180183
"operation": {"id": "Operation_Type=" + operation},
181-
"targetDataset": {"id": dataset_id},
184+
"targetDataset": {"id": table_id},
182185
"schema": schema,
183186
}
184187

@@ -282,55 +285,57 @@ def list_bucket(self, bucket_id=None):
282285
else:
283286
logging.warning("HTTP Error {}".format(r.status_code))
284287

285-
def list_dataset(self, dataset_id=None):
286-
"""Obtain details for all datasets or a given dataset.
288+
def list_table(self, table_name=None):
289+
"""Obtain details for all tables or a given table.
287290
288291
Parameters
289292
----------
290-
dataset_id : str
291-
The ID of the dataset to obtain details about. If the default value
292-
of None is specified, details regarding all datasets is returned.
293+
table_name : str
294+
The name of the table to obtain details about. If the default value
295+
of None is specified, details regarding first 100 tables is returned.
293296
294297
Returns
295298
-------
296299
If the request is successful, a dictionary containing information about
297-
the dataset is returned.
300+
the table is returned.
298301
299302
"""
300-
url = self.prism_endpoint + "/datasets"
303+
url = self.prism_endpoint + "/datasets?"
304+
305+
if table_name is not None:
306+
url = url + "name=" + table_name
301307

302-
if dataset_id is not None:
303-
url = url + "/" + dataset_id
308+
params = {"limit": 100}
304309

305310
headers = {"Authorization": "Bearer " + self.bearer_token}
306311

307-
r = requests.get(url, headers=headers)
312+
r = requests.get(url, params=params, headers=headers)
308313

309314
if r.status_code == 200:
310-
logging.info("Successfully obtained information about your datasets")
315+
logging.info("Successfully obtained information about your tables")
311316
return r.json()
312317
else:
313318
logging.warning("HTTP Error {}".format(r.status_code))
314319

315-
def describe_dataset(self, dataset_id=None):
316-
"""Obtain details for for a given dataset/table
320+
def describe_table(self, table_id=None):
321+
"""Obtain details for for a given table
317322
318323
Parameters
319324
----------
320-
dataset_id : str
321-
The ID of the dataset to obtain datails about. If the default value
322-
of None is specified, details regarding all datasets is returned.
325+
table_id : str
326+
The ID of the table to obtain details about. If the default value
327+
of None is specified, details regarding all tables is returned.
323328
324329
Returns
325330
-------
326331
If the request is successful, a dictionary containing information about
327-
the dataset is returned.
332+
the table is returned.
328333
329334
"""
330-
url = self.prism_endpoint + "/datasets"
335+
url = self.prism_endpoint + "/datasets/"
331336

332-
if dataset_id is not None:
333-
url = url + "/" + dataset_id + "/describe"
337+
if table_id is not None:
338+
url = url + table_id + "/describe"
334339

335340
headers = {"Authorization": "Bearer " + self.bearer_token}
336341

@@ -343,16 +348,16 @@ def describe_dataset(self, dataset_id=None):
343348
logging.warning("HTTP Error {}".format(r.status_code))
344349

345350
def convert_describe_schema_to_bucket_schema(self, describe_schema):
346-
"""Convert schema (derived from describe dataset/table) to bucket schema
351+
"""Convert schema (derived from describe table) to bucket schema
347352
348353
Parameters
349354
----------
350-
schema : dict
355+
describe_schema: dict
351356
A dictionary containing the describe schema for your dataset.
352357
353358
Returns
354359
-------
355-
If the request is succesful, a dictionary containing the bucket schema is returned.
360+
If the request is successful, a dictionary containing the bucket schema is returned.
356361
The results can then be passed to the create_bucket function
357362
358363
"""
@@ -362,6 +367,16 @@ def convert_describe_schema_to_bucket_schema(self, describe_schema):
362367
# in the dict that is in ['data'][0]
363368
fields = describe_schema["data"][0]["fields"]
364369

370+
# Create and assign useAsOperationKey field with true/false values based on externalId value
371+
operation_key_false = {"useAsOperationKey": False}
372+
operation_key_true = {"useAsOperationKey": True}
373+
374+
for i in fields:
375+
if i["externalId"] is True:
376+
i.update(operation_key_true)
377+
else:
378+
i.update(operation_key_false)
379+
365380
# Now trim our fields data to keep just what we need
366381
for i in fields:
367382
del i["id"]
@@ -385,9 +400,9 @@ def convert_describe_schema_to_bucket_schema(self, describe_schema):
385400
}
386401

387402
# The footer for the load schema
388-
schemaVersion = {"id": "Schema_Version=1.0"}
403+
schema_version = {"id": "Schema_Version=1.0"}
389404

390405
bucket_schema["fields"] = fields
391-
bucket_schema["schemaVersion"] = schemaVersion
406+
bucket_schema["schemaVersion"] = schema_version
392407

393408
return bucket_schema

0 commit comments

Comments
 (0)