Skip to content

Commit

Permalink
Improve extract performance via ignoring directories early during os.…
Browse files Browse the repository at this point in the history
…walk

Co-authored-by: Steven Kao <st.kao.05@gmail.com>
  • Loading branch information
akx and stkao05 committed Feb 7, 2023
1 parent 08af5e2 commit fc8fca4
Showing 1 changed file with 29 additions and 8 deletions.
37 changes: 29 additions & 8 deletions babel/messages/extract.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,10 +102,25 @@ def _strip(line: str):
comments[:] = map(_strip, comments)


def default_directory_filter(dirpath: str | os.PathLike[str]) -> bool:
subdir = os.path.basename(dirpath)
# Legacy default behavior: ignore dot and underscore directories
return not (subdir.startswith('.') or subdir.startswith('_'))
def make_default_directory_filter(
method_map: Iterable[tuple[str, str]],
root_dir: str | os.PathLike[str],
):
def directory_filter(dirpath: str | os.PathLike[str]) -> bool:
subdir = os.path.basename(dirpath)
# Legacy default behavior: ignore dot and underscore directories
if subdir.startswith('.') or subdir.startswith('_'):
return False

dir_rel = os.path.relpath(dirpath, root_dir).replace(os.sep, '/')

for pattern, method in method_map:
if method == "ignore" and pathmatch(pattern, dir_rel):
return False

Check warning on line 119 in babel/messages/extract.py

View check run for this annotation

Codecov / codecov/patch

babel/messages/extract.py#L119

Added line #L119 was not covered by tests

return True

return directory_filter


def extract_from_dir(
Expand Down Expand Up @@ -189,13 +204,19 @@ def extract_from_dir(
"""
if dirname is None:
dirname = os.getcwd()

if options_map is None:
options_map = {}

dirname = os.path.abspath(dirname)

if directory_filter is None:
directory_filter = default_directory_filter
directory_filter = make_default_directory_filter(
method_map=method_map,
root_dir=dirname,
)

absname = os.path.abspath(dirname)
for root, dirnames, filenames in os.walk(absname):
for root, dirnames, filenames in os.walk(dirname):
dirnames[:] = [
subdir for subdir in dirnames
if directory_filter(os.path.join(root, subdir))
Expand All @@ -213,7 +234,7 @@ def extract_from_dir(
keywords,
comment_tags,
strip_comment_tags,
dirpath=absname,
dirpath=dirname,
)


Expand Down

0 comments on commit fc8fca4

Please sign in to comment.