Skip to content

feat: add compound file ordinal endpoints update #56

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Feb 7, 2023
Merged

feat: add compound file ordinal endpoints update #56

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
122 changes: 104 additions & 18 deletions sdk/diffgram/file/compound_file.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@


class CompoundChildFile:
id: int
path: str
url: str
blob_path: str
Expand All @@ -24,8 +25,10 @@ class CompoundChildFile:
video_split_duration: int
local_id: str
ordinal: int
root_file: 'CompoundFile'

def __init__(self,
root_file: 'CompoundFile',
child_file_type: str,
path: str = None,
url: str = None,
Expand All @@ -42,7 +45,9 @@ def __init__(self,
assume_new_instances_machine_made: bool = None,
convert_names_to_label_files: bool = None,
video_split_duration: int = None,
ordinal: int = 0):
ordinal: int = 0,
id: int = None):
self.root_file = root_file
self.child_file_type = child_file_type
self.path = path
self.url = url
Expand All @@ -61,6 +66,38 @@ def __init__(self,
self.video_split_duration = video_split_duration
self.local_id = str(uuid4())
self.ordinal = ordinal
self.id = None

def __str__(self):
return f'<CompoundChildFile id={self.id} ordinal={self.ordinal} file_name={self.file_name} child_file_type={self.child_file_type}>'
def refresh_from_data_dict(self, data: dict):
if not data:
return
for key in data:
setattr(self, key, data[key])

def update(self):
"""
Syncs child file data with backend API
:return:
"""
payload = {
'ordinal': self.ordinal,
}
client = self.root_file.project
endpoint = f"/api/v1/project/{self.root_file.project.project_string_id}/file/{self.id}/update-metadata"

response = client.session.put(
client.host + endpoint,
json = payload)

client.handle_errors(response)

data = response.json()
new_file_data = data['file']

self.refresh_from_data_dict(data = new_file_data)
return self

def set_ordinal(self, value: int):
self.ordinal = value
Expand All @@ -69,23 +106,57 @@ def set_ordinal(self, value: int):
class CompoundFile:
project: Project
parent_file_data: dict
child_files_to_upload: List[CompoundChildFile]
child_files: List[CompoundChildFile]

def __init__(self, project: Project, name: str, directory_id: int):
self.project = project
self.name = name
self.directory_id = directory_id
self.child_files_to_upload = []
self.child_files = []

@staticmethod
def from_dict(project: Project, dir_id: int, dict_data: dict):
result = CompoundFile(project = project, name = dict_data.get('original_filename'), directory_id = dir_id)
result.__refresh_compound_file_from_data_dict(data = dict_data)
child_files = result.__fetch_child_files()
result.child_files = child_files
return result

def __fetch_child_files(self) -> List[CompoundChildFile]:
client = self.project
endpoint = f"/api/v1/project/{self.project.project_string_id}/file/{self.id}/child-files"

response = client.session.get(client.host + endpoint)

client.handle_errors(response)

data = response.json()
child_files_data = data['child_files']
result = []
for elm in child_files_data:
child_file = CompoundChildFile(root_file = self, child_file_type = elm.get('type'))
child_file.refresh_from_data_dict(data = elm)
result.append(child_file)
return result

def update_all(self) -> bool:
"""
Syncs parent and child metadata with backend API.
:return: True/False
"""
for child in self.child_files:
child.update()
return True

def __refresh_compound_file_from_data_dict(self, data: dict):
if not data:
return
for key in data:
setattr(self, key, data[key])

def remove_compound_file(self, child_file: CompoundChildFile) -> List[CompoundChildFile]:
self.child_files_to_upload.remove(child_file)
return self.child_files_to_upload
def remove_child_file(self, child_file: CompoundChildFile) -> List[CompoundChildFile]:
self.child_files.remove(child_file)
return self.child_files

def __create_compound_parent_file(self):
url = f'/api/v1/project/{self.project.project_string_id}/file/new-compound'
Expand All @@ -110,7 +181,8 @@ def __create_child_file(self, child_file: CompoundChildFile):
frame_packet_map = child_file.frame_packet_map,
assume_new_instances_machine_made = child_file.assume_new_instances_machine_made,
convert_names_to_label_files = child_file.convert_names_to_label_files,
parent_file_id = self.parent_file_data.get('id')
parent_file_id = self.parent_file_data.get('id'),
ordinal = child_file.ordinal,
)
elif child_file.child_file_type == 'from_url':
return self.project.file.from_url(
Expand All @@ -121,7 +193,8 @@ def __create_child_file(self, child_file: CompoundChildFile):
video_split_duration = child_file.video_split_duration,
instance_list = child_file.instance_list,
frame_packet_map = child_file.frame_packet_map,
parent_file_id = self.parent_file_data.get('id')
parent_file_id = self.parent_file_data.get('id'),
ordinal = child_file.ordinal,
)
elif child_file.child_file_type == 'from_blob_path':
return self.project.file.from_blob_path(
Expand All @@ -132,7 +205,8 @@ def __create_child_file(self, child_file: CompoundChildFile):
instance_list = child_file.instance_list,
file_name = child_file.file_name,
frame_packet_map = child_file.frame_packet_map,
parent_file_id = self.parent_file_data.get('id')
parent_file_id = self.parent_file_data.get('id'),
ordinal = child_file.ordinal,
)

def add_child_from_local(self,
Expand All @@ -141,17 +215,21 @@ def add_child_from_local(self,
frame_packet_map: dict = None,
assume_new_instances_machine_made: bool = True,
convert_names_to_label_files: bool = True,
ordinal: int = 0):
ordinal: int = None):
if ordinal is None:
ordinal = len(self.child_files)
new_child_file = CompoundChildFile(
root_file = self,
child_file_type = "from_local",
path = path,
directory_id = self.directory_id,
instance_list = instance_list,
frame_packet_map = frame_packet_map,
assume_new_instances_machine_made = assume_new_instances_machine_made,
convert_names_to_label_files = convert_names_to_label_files
convert_names_to_label_files = convert_names_to_label_files,
ordinal = ordinal
)
self.child_files_to_upload.append(new_child_file)
self.child_files.append(new_child_file)
return new_child_file

def add_child_file_from_url(self,
Expand All @@ -162,8 +240,11 @@ def add_child_file_from_url(self,
video_split_duration: int = None,
instance_list: list = None,
frame_packet_map: dict = None,
ordinal: int = 0):
ordinal: int = None):
if ordinal is None:
ordinal = len(self.child_files)
new_child_file = CompoundChildFile(
root_file = self,
child_file_type = "from_url",
url = url,
media_type = media_type,
Expand All @@ -173,8 +254,9 @@ def add_child_file_from_url(self,
video_split_duration = video_split_duration,
instance_list = instance_list,
frame_packet_map = frame_packet_map,
ordinal = ordinal
)
self.child_files_to_upload.append(new_child_file)
self.child_files.append(new_child_file)
return new_child_file

def add_child_from_blob_path(self,
Expand All @@ -185,9 +267,12 @@ def add_child_from_blob_path(self,
instance_list: list = None,
file_name: str = None,
frame_packet_map: dict = None,
ordinal: int = 0
ordinal: int = None
):
if ordinal is None:
ordinal = len(self.child_files)
new_child_file = CompoundChildFile(
root_file = self,
child_file_type = "from_blob_path",
blob_path = blob_path,
bucket_name = bucket_name,
Expand All @@ -197,15 +282,16 @@ def add_child_from_blob_path(self,
instance_list = instance_list,
file_name = file_name,
frame_packet_map = frame_packet_map,
ordinal = ordinal
)
self.child_files_to_upload.append(new_child_file)
self.child_files.append(new_child_file)
return new_child_file

def upload(self):
if len(self.child_files_to_upload) == 0:
if len(self.child_files) == 0:
raise AssertionError('Need to add at least one child file to the compound file before calling upload()')
parent_file_data: dict = self.__create_compound_parent_file()

for child_file in self.child_files_to_upload:
for child_file in self.child_files:
self.__create_child_file(child_file)
return parent_file_data
37 changes: 27 additions & 10 deletions sdk/diffgram/file/file_constructor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,8 @@ def from_local(
frame_packet_map: dict = None,
assume_new_instances_machine_made: bool = True,
convert_names_to_label_files: bool = True,
parent_file_id: int = None
parent_file_id: int = None,
ordinal: int = 0
):
"""
Create a Project file from local path
Expand All @@ -56,7 +57,9 @@ def from_local(

files = {'file': (os.path.basename(path), open(path, 'rb'), 'application/octet-stream')}

json_payload = {}
json_payload = {
'ordinal': ordinal
}

if directory_id is None:
directory_id = self.client.directory_id
Expand Down Expand Up @@ -107,13 +110,15 @@ def __build_packet_payload(self,
bucket_name: str = None,
file_name: str = None,
blob_path: str = None,
parent_file_id: int = None):
parent_file_id: int = None,
ordinal: int = 0):
packet = {'media': {}}
packet['media']['url'] = url
packet['media']['type'] = media_type

# Existing Instances
packet['frame_packet_map'] = frame_packet_map
packet['ordinal'] = ordinal
packet['type'] = type
packet['connection_id'] = connection_id
packet['parent_file_id'] = parent_file_id
Expand All @@ -140,7 +145,8 @@ def from_blob_path(self,
instance_list: list = None,
file_name: str = None,
frame_packet_map: dict = None,
parent_file_id: int = None):
parent_file_id: int = None,
ordinal: int = 0):
"""
Bind a blob path in the given connection ID into Diffgram
:param blob_path:
Expand All @@ -167,7 +173,8 @@ def from_blob_path(self,
file_name = name,
directory_id = directory_id,
type = "from_blob_path",
parent_file_id=parent_file_id
parent_file_id=parent_file_id,
ordinal = ordinal
)
self.from_packet(packet = packet)
return True
Expand All @@ -181,7 +188,8 @@ def from_url(
video_split_duration: int = None,
instance_list: list = None, # for Images
frame_packet_map: dict = None, # for Video
parent_file_id: int = None
parent_file_id: int = None,
ordinal: int = 0
):
"""

Expand Down Expand Up @@ -214,7 +222,8 @@ def from_url(
video_split_duration = video_split_duration,
instance_list = instance_list,
frame_packet_map = frame_packet_map,
parent_file_id = parent_file_id
parent_file_id = parent_file_id,
ordinal = ordinal
)
self.from_packet(packet = packet)

Expand Down Expand Up @@ -554,7 +563,15 @@ def get_by_id(self,

response_json = response.json()
file_data = response_json.get(file_response_key)
if file_data.get('type') == 'compound':
from diffgram.file.compound_file import CompoundFile
return CompoundFile.from_dict(
project = self.client,
dir_id = self.client.directory_id,
dict_data = file_data
)

return File.new(
client = self.client,
file_json = file_data)
else:
return File.new(
client = self.client,
file_json = file_data)