Skip to content

Commit

Permalink
Rewrite conversion of relative and absolute path support in OpenShot (O…
Browse files Browse the repository at this point in the history 
…penShot#2477)

* Convert transition paths for redo undo history loading and saving. This is to address absolute paths burried inside the undo/redo history.

* Complete rewrite of relative and absolute path conversions using regex when the JSON is read and written to the file. This prevents lots of iterating over dicts, and is more full-proof for finding paths, even ones listed in the undo/redo history.

* Remove any update history with a key == "history", which should not even happen anymore... but some older project files can still contain these keys, which can be giant blobs of data.
  • Loading branch information
@jonoomph
jonoomph authored Dec 27, 2018
1 parent ff80496 commit d7fe16c
Show file tree
Hide file tree
Showing 3 changed files with 95 additions and 154 deletions.
82 changes: 76 additions & 6 deletions src/classes/json_data.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,14 @@
import simplejson as json

import copy
import os
import re

from classes.logger import log
from classes import info

# Compiled path regex
path_regex = re.compile(r'\"(?:image|path)\":.*?\"(.*?)\"')


class JsonDataStore:
Expand Down Expand Up @@ -119,14 +125,15 @@ def merge_settings(self, default, user):
# Return merged dictionary
return user

def read_from_file(self, file_path):
def read_from_file(self, file_path, path_mode="ignore"):
""" Load JSON settings from a file """
# log.debug("loading {}".format(file_path))
try:
with open(file_path, 'r') as f:
contents = f.read()
if contents:
# log.debug("loaded", contents)
if path_mode == "absolute":
# Convert any paths to absolute
contents = self.convert_paths_to_absolute(file_path, contents)
return json.loads(contents)
except Exception as ex:
msg = ("Couldn't load {} file: {}".format(self.data_type, ex))
Expand All @@ -136,13 +143,76 @@ def read_from_file(self, file_path):
log.warning(msg)
raise Exception(msg)

def write_to_file(self, file_path, data):
def write_to_file(self, file_path, data, path_mode="ignore", previous_path=None):
""" Save JSON settings to a file """
# log.debug(json.dumps(data))
try:
contents = json.dumps(data)
if path_mode == "relative":
# Convert any paths to relative
contents = self.convert_paths_to_relative(file_path, previous_path, contents)
with open(file_path, 'w') as f:
f.write(json.dumps(data))
f.write(contents)
except Exception as ex:
msg = ("Couldn't save {} file:\n{}\n{}".format(self.data_type, file_path, ex))
log.error(msg)
raise Exception(msg)

def convert_paths_to_absolute(self, file_path, data):
""" Convert all paths to absolute using regex """
try:
# Get project folder
existing_project_folder = os.path.dirname(file_path)

# Find all "path" attributes in the JSON string
for path in path_regex.findall(data):
# Find absolute path of file (if needed)
if "@transitions" not in path and not os.path.isabs(path):
# Convert path to the correct relative path (based on the existing folder)
new_path = os.path.abspath(os.path.join(existing_project_folder, path))
data = data.replace('"%s"' % path, '"%s"' % new_path)

# Determine if @transitions path is found
elif "@transitions" in path:
new_path = path.replace("@transitions", os.path.join(info.PATH, "transitions"))
data = data.replace('"%s"' % path, '"%s"' % new_path)

except Exception as ex:
log.error("Error while converting relative paths to absolute paths: %s" % str(ex))

return data

def convert_paths_to_relative(self, file_path, previous_path, data):
""" Convert all paths relative to this filepath """
try:
# Get project folder
new_project_folder = os.path.dirname(file_path)
existing_project_folder = os.path.dirname(file_path)
if previous_path:
existing_project_folder = os.path.dirname(previous_path)

# Find all "path" attributes in the JSON string
for path in path_regex.findall(data):
folder_path, file_path = os.path.split(path)

# Find absolute path of file (if needed)
if not os.path.join(info.PATH, "transitions") in folder_path:
# Convert path to the correct relative path (based on the existing folder)
orig_abs_path = path
if not os.path.isabs(path):
orig_abs_path = os.path.abspath(os.path.join(existing_project_folder, path))
new_rel_path = os.path.relpath(orig_abs_path, new_project_folder)
data = data.replace('"%s"' % path, '"%s"' % new_rel_path)

# Determine if @transitions path is found
else:
# Yes, this is an OpenShot transitions
folder_path, category_path = os.path.split(folder_path)

# Convert path to @transitions/ path
new_path = os.path.join("@transitions", category_path, file_path)
data = data.replace('"%s"' % path, '"%s"' % new_path)

except Exception as ex:
log.error("Error while converting absolute paths to relative paths: %s" % str(ex))

return data
147 changes: 6 additions & 141 deletions src/classes/project_data.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,16 +27,16 @@
along with OpenShot Library. If not, see <http://www.gnu.org/licenses/>.
"""

import copy
import glob
import os
import random
import copy
import shutil
import glob

from classes.json_data import JsonDataStore
from classes.updates import UpdateInterface
from classes import info, settings
from classes.json_data import JsonDataStore
from classes.logger import log
from classes.updates import UpdateInterface


class ProjectDataStore(JsonDataStore, UpdateInterface):
Expand Down Expand Up @@ -316,7 +316,7 @@ def load(self, file_path):

try:
# Attempt to load v2.X project file
project_data = self.read_from_file(file_path)
project_data = self.read_from_file(file_path, path_mode="absolute")

except Exception as ex:
try:
Expand All @@ -333,9 +333,6 @@ def load(self, file_path):
# On success, save current filepath
self.current_filepath = file_path

# Convert all paths back to absolute
self.convert_paths_to_absolute()

# Check if paths are all valid
self.check_if_paths_are_valid()

Expand Down Expand Up @@ -695,25 +692,17 @@ def save(self, file_path, move_temp_files=True, make_paths_relative=True):
if move_temp_files:
self.move_temp_paths_to_project_folder(file_path)

# Convert all file paths to relative based on this new project file's directory
if make_paths_relative:
self.convert_paths_to_relative(file_path)

# Append version info
v = openshot.GetVersion()
self._data["version"] = { "openshot-qt" : info.VERSION,
"libopenshot" : v.ToString() }

# Try to save project settings file, will raise error on failure
self.write_to_file(file_path, self._data)
self.write_to_file(file_path, self._data, path_mode="relative", previous_path=self.current_filepath)

# On success, save current filepath
self.current_filepath = file_path

# Convert all paths back to absolute
if make_paths_relative:
self.convert_paths_to_absolute()

# Add to recent files setting
self.add_to_recent_files(file_path)

Expand Down Expand Up @@ -829,72 +818,6 @@ def add_to_recent_files(self, file_path):
s.set("recent_projects", recent_projects)
s.save()

def convert_paths_to_relative(self, file_path):
""" Convert all paths relative to this filepath """
try:
# Get project folder
existing_project_folder = None
if self.current_filepath:
existing_project_folder = os.path.dirname(self.current_filepath)
new_project_folder = os.path.dirname(file_path)

# Loop through each file
for file in self._data["files"]:
path = file["path"]
# Find absolute path of file (if needed)
if not os.path.isabs(path):
# Convert path to the correct relative path (based on the existing folder)
path = os.path.abspath(os.path.join(existing_project_folder, path))

# Convert absolute path to relavite
file["path"] = os.path.relpath(path, new_project_folder)

# Loop through each clip
for clip in self._data["clips"]:
# Update reader path
path = clip["reader"]["path"]
# Find absolute path of file (if needed)
if not os.path.isabs(path):
# Convert path to the correct relative path (based on the existing folder)
path = os.path.abspath(os.path.join(existing_project_folder, path))
# Convert absolute path to relavite
clip["reader"]["path"] = os.path.relpath(path, new_project_folder)

# Update clip image path
path = clip["image"]
# Find absolute path of file (if needed)
if not os.path.isabs(path):
# Convert path to the correct relative path (based on the existing folder)
path = os.path.abspath(os.path.join(existing_project_folder, path))
# Convert absolute path to relavite
clip["image"] = os.path.relpath(path, new_project_folder)

# Loop through each transition
for effect in self._data["effects"]:
# Update reader path
path = effect["reader"]["path"]

# Determine if this path is the official transition path
folder_path, file_path = os.path.split(path)
if os.path.join(info.PATH, "transitions") in folder_path:
# Yes, this is an OpenShot transitions
folder_path, category_path = os.path.split(folder_path)

# Convert path to @transitions/ path
effect["reader"]["path"] = os.path.join("@transitions", category_path, file_path)
continue

# Find absolute path of file (if needed)
if not os.path.isabs(path):
# Convert path to the correct relative path (based on the existing folder)
path = os.path.abspath(os.path.join(existing_project_folder, path))
# Convert absolute path to relavite
effect["reader"]["path"] = os.path.relpath(path, new_project_folder)

except Exception as ex:
log.error("Error while converting absolute paths to relative paths: %s" % str(ex))


def check_if_paths_are_valid(self):
"""Check if all paths are valid, and prompt to update them if needed"""
# Get import path or project folder
Expand Down Expand Up @@ -971,64 +894,6 @@ def check_if_paths_are_valid(self):
self._data["clips"].remove(clip)
break

def convert_paths_to_absolute(self):
""" Convert all paths to absolute """
try:
# Get project folder
existing_project_folder = None
if self.current_filepath:
existing_project_folder = os.path.dirname(self.current_filepath)

# Loop through each file
for file in self._data["files"]:
path = file["path"]
# Find absolute path of file (if needed)
if not os.path.isabs(path):
# Convert path to the correct relative path (based on the existing folder)
path = os.path.abspath(os.path.join(existing_project_folder, path))

# Convert absolute path to relavite
file["path"] = path

# Loop through each clip
for clip in self._data["clips"]:
# Update reader path
path = clip["reader"]["path"]
# Find absolute path of file (if needed)
if not os.path.isabs(path):
# Convert path to the correct relative path (based on the existing folder)
path = os.path.abspath(os.path.join(existing_project_folder, path))
# Convert absolute path to relavite
clip["reader"]["path"] = path

# Update clip image path
path = clip["image"]
# Find absolute path of file (if needed)
if not os.path.isabs(path):
# Convert path to the correct relative path (based on the existing folder)
path = os.path.abspath(os.path.join(existing_project_folder, path))
# Convert absolute path to relavite
clip["image"] = path

# Loop through each transition
for effect in self._data["effects"]:
# Update reader path
path = effect["reader"]["path"]

# Determine if @transitions path is found
if "@transitions" in path:
path = path.replace("@transitions", os.path.join(info.PATH, "transitions"))

# Find absolute path of file (if needed)
if not os.path.isabs(path):
# Convert path to the correct relative path (based on the existing folder)
path = os.path.abspath(os.path.join(existing_project_folder, path))
# Convert absolute path to relavite
effect["reader"]["path"] = path

except Exception as ex:
log.error("Error while converting relative paths to absolute paths: %s" % str(ex))

def changed(self, action):
""" This method is invoked by the UpdateManager each time a change happens (i.e UpdateInterface) """
# Track unsaved changes
Expand Down
20 changes: 13 additions & 7 deletions src/classes/updates.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,9 @@
"""

from classes.logger import log
from classes import info
import copy
import os

try:
import json
Expand Down Expand Up @@ -145,15 +147,16 @@ def load_history(self, project):
history = project.get(["history"])

# Loop through each, and load serialized data into updateAction objects
# Ignore any load actions or history update actions
for actionDict in history.get("redo", []):
action = UpdateAction()
action.load_json(json.dumps(actionDict))
if action.type != "load":
if action.type != "load" and action.key[0] != "history":
self.redoHistory.append(action)
for actionDict in history.get("undo", []):
action = UpdateAction()
action.load_json(json.dumps(actionDict))
if action.type != "load":
if action.type != "load" and action.key[0] != "history":
self.actionHistory.append(action)

# Notify watchers of new status
Expand All @@ -164,14 +167,17 @@ def save_history(self, project, history_length):
redo_list = []
undo_list = []

# Loop through each, and serialize
# Loop through each updateAction object and serialize
# Ignore any load actions or history update actions
history_length_int = int(history_length)
for action in self.redoHistory[-history_length_int:]:
if action.type != "load":
redo_list.append(json.loads(action.json()))
if action.type != "load" and action.key[0] != "history":
actionDict = json.loads(action.json())
redo_list.append(actionDict)
for action in self.actionHistory[-history_length_int:]:
if action.type != "load":
undo_list.append(json.loads(action.json()))
if action.type != "load" and action.key[0] != "history":
actionDict = json.loads(action.json())
undo_list.append(actionDict)

# Set history data in project
self.ignore_history = True
Expand Down

0 comments on commit d7fe16c

Please sign in to comment.