Skip to content

Refactor Ports.fetch_port to avoid locking the cache where possible #13321

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jan 26, 2021
Merged

Refactor Ports.fetch_port to avoid locking the cache where possible #13321

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 14 additions & 19 deletions tools/system_libs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1696,10 +1696,6 @@ def fetch_project(name, url, subdir, is_tarbz2=False, sha512hash=None):
logger.debug(' (at ' + fullname + ')')
Ports.name_cache.add(name)

class State(object):
retrieved = False
unpacked = False

def retrieve():
# retrieve from remote server
logger.info('retrieving port: ' + name + ' from ' + url)
Expand All @@ -1724,7 +1720,6 @@ def retrieve():
shared.exit_with_error('Unexpected hash: ' + actual_hash + '\n'
'If you are updating the port, please update the hash in the port module.')
open(fullpath, 'wb').write(data)
State.retrieved = True

def check_tag():
if is_tarbz2:
Expand Down Expand Up @@ -1755,28 +1750,28 @@ def unpack():
with utils.chdir(fullname):
z.extractall()

State.unpacked = True
# before acquiring the lock we have an early out if the port already exists
if os.path.exists(fullpath) and check_tag():
return

# main logic. do this under a cache lock, since we don't want multiple jobs to
# retrieve the same port at once

with shared.Cache.lock():
if not os.path.exists(fullpath):
retrieve()

if not os.path.exists(fullname):
unpack()

if not check_tag():
if os.path.exists(fullpath):
# Another early out in case another process build the library while we were
# waiting for the lock
if check_tag():
return
# file exists but tag is bad
logger.warning('local copy of port is not correct, retrieving from remote server')
shared.try_delete(fullname)
shared.try_delete(fullpath)
retrieve()
unpack()

if State.unpacked:
# we unpacked a new version, clear the build in the cache
Ports.clear_project_build(name)
retrieve()
unpack()

# we unpacked a new version, clear the build in the cache
Ports.clear_project_build(name)

@staticmethod
def clear_project_build(name):
Expand Down