Skip to content

Commit

Permalink
erorr handling, moved the validation out of upload process
Browse files Browse the repository at this point in the history
  • Loading branch information
@arjunprakash027
arjunprakash027 committed May 19, 2024
1 parent 920a94a commit 8679a28
Show file tree
Hide file tree
Showing 3 changed files with 79 additions and 62 deletions.
46 changes: 45 additions & 1 deletion airlift/airtable_client.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,15 @@ def missing_field_single(self,field:str):
return False


def missing_fields_check(self,data:ATDATATYPE,disable_bypass:bool,ignore_columns:List[str]):
def _missing_fields_check(self,data:ATDATATYPE,args:dict):

disable_bypass = args.disable_bypass_column_creation

if args.rename_key_column:
ignore_columns = [args.rename_key_column[0]]
else:
ignore_columns = None

airtable_table_fields = []
user_csv_fields = []

Expand Down Expand Up @@ -124,6 +131,43 @@ def _create_new_field(self,field_name:str) -> None:

else:
logger.warning(f"unknown error : {response.text}")



def _rename_key_column_check(self,args:dict) -> None:

if args.rename_key_column[1] == args.rename_key_column[0]:
raise CriticalError("rename-key-column arguement has same column name!")

def create_uploadable_data(self,data:ATDATATYPE,args:dict):


data = self._missing_fields_check(data,args)

#validation and creating new data fields
if args.rename_key_column:
self._rename_key_column_check(args=args)

if args.attachment_columns_map:
for attachment in args.attachment_columns_map:
data['fields'][attachment[1]] = ""

if args.columns_copy:
for column in args.columns_copy[1::]:
if self.missing_field_single(column):
data['fields'][column] = data['fields'][args.columns_copy[0]]
else:
raise CriticalError(f"The Column {column} is not present in airtable! Please create it and try again")

if args.rename_key_column:
if self.missing_field_single(args.rename_key_column[1]):
data['fields'][args.rename_key_column[1]] = data['fields'][args.rename_key_column[0]]
del data['fields'][args.rename_key_column[0]]
else:
raise CriticalError(f"The Key Column {args.rename_key_column[1]} is not present in airtable! Please create it and try again")

return data




86 changes: 32 additions & 54 deletions airlift/airtable_upload.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,71 +64,49 @@ def _worker(self,data_queue: Queue, progress_bar) -> None:
while True:
try:
data = data_queue.get_nowait()

if self.attachment_columns_map:
for attachment in self.attachment_columns_map:
data['fields'][attachment[1]] = ""

if self.columns_copy:
for column in self.columns_copy[1::]:
if self.client.missing_field_single(column):
data['fields'][column] = data['fields'][self.columns_copy[0]]
else:
raise CriticalError(f"The Column {column} is not present in airtable! Please create it and try again")

if self.rename_key_column:
if self.client.missing_field_single(self.rename_key_column[1]):
data['fields'][self.rename_key_column[1]] = data['fields'][self.rename_key_column[0]]
del data['fields'][self.rename_key_column[0]]
else:
raise CriticalError(f"The Key Column {self.rename_key_column[1]} is not present in airtable! Please create it and try again")

try:
for key, value in data['fields'].items():
if self.attachment_columns:
if self.dbx:
if key in self.attachment_columns:
for key, value in data['fields'].items():
if self.attachment_columns:
if self.dbx:
if key in self.attachment_columns:
try:
if self.dirname:
data['fields'][key] = [{"url": self.dbx.upload_to_dropbox(f"{self.dirname}/{value}")}]
else:
data['fields'][key] = [{"url": self.dbx.upload_to_dropbox(f"{value}")}]

except Exception as e:
self.write_log(self.log,f"{value} Could not be found!")
tqdm.write(f"{value} Could not be found!")
data['fields'][key] = ""

if self.attachment_columns_map:
if self.dbx:
for attachments in self.attachment_columns_map:
if key == attachments[0]:
try:
if self.dirname:
data['fields'][key] = [{"url": self.dbx.upload_to_dropbox(f"{self.dirname}/{value}")}]
else:
data['fields'][key] = [{"url": self.dbx.upload_to_dropbox(f"{value}")}]

data['fields'][attachments[1]] = [
{"url": self.dbx.upload_to_dropbox(f"{self.dirname}/{value}")}]
else:
data['fields'][attachments[1]] = [
{"url": self.dbx.upload_to_dropbox(f"{value}")}]
except Exception as e:
self.write_log(self.log,f"{value} Could not be found!")
tqdm.write(f"{value} Could not be found!")
data['fields'][key] = ""

if self.attachment_columns_map:
if self.dbx:
for attachments in self.attachment_columns_map:
if key == attachments[0]:
try:
if self.dirname:
data['fields'][attachments[1]] = [
{"url": self.dbx.upload_to_dropbox(f"{self.dirname}/{value}")}]
else:
data['fields'][attachments[1]] = [
{"url": self.dbx.upload_to_dropbox(f"{value}")}]
except Exception as e:
self.write_log(self.log,f"{value} Could not be found!")
tqdm.write(f"{value} Could not be found!")
data['fields'][attachments[1]] = ""
#ic(data['fields'])
data['fields'][attachments[1]] = ""
#ic(data['fields'])

else:
logger.error("Dropbox token not provided! Aborting the upload!")

self.client.single_upload(data)
progress_bar.update(1)
except Exception as e:
logger.error(e)
else:
logger.error("Dropbox token not provided! Aborting the upload!")

self.client.single_upload(data)
progress_bar.update(1)
except Exception as e:

if data_queue.empty():
break
else:
logger.error(e)
raise CriticalError



9 changes: 2 additions & 7 deletions airlift/cli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,13 +59,8 @@ def cli(*argv: str) -> None:
if not data:
raise CriticalError("File is empty!")

#checking for missing columns
if args.rename_key_column:
ignore_column_check = [args.rename_key_column[0]]
else:
ignore_column_check = None

data = airtable_client.missing_fields_check(data,disable_bypass=args.disable_bypass_column_creation,ignore_columns=ignore_column_check)
#validating data and creating an uploadable data
data = airtable_client.create_uploadable_data(data=data,args=args)

#uploading the data
upload_instance = Upload(client=airtable_client, new_data=data,dbx=dbx,args=args)
Expand Down

0 comments on commit 8679a28

Please sign in to comment.