Skip to content

Commit eb10d1f

Browse files
mattdangerwmattdangerw
authored
Redundant imports; no path hacking in package (keras-team#21187)
1 parent 41d7737 commit eb10d1f

File tree

135 files changed

+3632
-2813
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

135 files changed

+3632
-2813
lines changed

‎.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ obj/**
1313
tmp/**
1414
.vs/
1515
dist/**
16-
*.egg-info/*
16+
**/*.egg-info/*
1717
.vscode
1818
examples/**/*.jpg
1919
.python-version

‎api_gen.py‎

Lines changed: 32 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
It generates API and formats user and generated APIs.
77
"""
88

9-
import importlib
109
import os
1110
import re
1211
import shutil
@@ -24,38 +23,38 @@ def ignore_files(_, filenames):
2423
def copy_source_to_build_directory(root_path):
2524
# Copy sources (`keras/` directory and setup files) to build dir
2625
build_dir = os.path.join(root_path, BUILD_DIR_NAME)
26+
build_package_dir = os.path.join(build_dir, PACKAGE)
27+
build_src_dir = os.path.join(build_package_dir, "src")
28+
root_src_dir = os.path.join(root_path, PACKAGE, "src")
2729
if os.path.exists(build_dir):
2830
shutil.rmtree(build_dir)
29-
os.mkdir(build_dir)
30-
shutil.copytree(
31-
PACKAGE, os.path.join(build_dir, PACKAGE), ignore=ignore_files
32-
)
31+
os.makedirs(build_package_dir)
32+
shutil.copytree(root_src_dir, build_src_dir)
3333
return build_dir
3434

3535

3636
def create_legacy_directory(package_dir):
3737
src_dir = os.path.join(package_dir, "src")
38-
api_dir = os.path.join(package_dir, "api")
3938
# Make keras/_tf_keras/ by copying keras/
40-
tf_keras_dirpath_parent = os.path.join(api_dir, "_tf_keras")
39+
tf_keras_dirpath_parent = os.path.join(package_dir, "_tf_keras")
4140
tf_keras_dirpath = os.path.join(tf_keras_dirpath_parent, "keras")
4241
os.makedirs(tf_keras_dirpath, exist_ok=True)
4342
with open(os.path.join(tf_keras_dirpath_parent, "__init__.py"), "w") as f:
44-
f.write("from keras.api._tf_keras import keras\n")
45-
with open(os.path.join(api_dir, "__init__.py")) as f:
43+
f.write("from keras._tf_keras import keras\n")
44+
with open(os.path.join(package_dir, "__init__.py")) as f:
4645
init_file = f.read()
4746
init_file = init_file.replace(
48-
"from keras.api import _legacy",
49-
"from keras.api import _tf_keras",
47+
"from keras import _legacy as _legacy",
48+
"from keras import _tf_keras as _tf_keras",
5049
)
51-
with open(os.path.join(api_dir, "__init__.py"), "w") as f:
50+
with open(os.path.join(package_dir, "__init__.py"), "w") as f:
5251
f.write(init_file)
5352
# Remove the import of `_tf_keras` in `keras/_tf_keras/keras/__init__.py`
54-
init_file = init_file.replace("from keras.api import _tf_keras\n", "\n")
53+
init_file = init_file.replace("from keras import _tf_keras\n", "\n")
5554
with open(os.path.join(tf_keras_dirpath, "__init__.py"), "w") as f:
5655
f.write(init_file)
57-
for dirname in os.listdir(api_dir):
58-
dirpath = os.path.join(api_dir, dirname)
56+
for dirname in os.listdir(package_dir):
57+
dirpath = os.path.join(package_dir, dirname)
5958
if os.path.isdir(dirpath) and dirname not in (
6059
"_legacy",
6160
"_tf_keras",
@@ -81,7 +80,7 @@ def create_legacy_directory(package_dir):
8180
for path in os.listdir(os.path.join(src_dir, "legacy"))
8281
if os.path.isdir(os.path.join(src_dir, "legacy", path))
8382
]
84-
for root, _, fnames in os.walk(os.path.join(api_dir, "_legacy")):
83+
for root, _, fnames in os.walk(os.path.join(package_dir, "_legacy")):
8584
for fname in fnames:
8685
if fname.endswith(".py"):
8786
legacy_fpath = os.path.join(root, fname)
@@ -95,22 +94,22 @@ def create_legacy_directory(package_dir):
9594
with open(legacy_fpath) as f:
9695
legacy_contents = f.read()
9796
legacy_contents = legacy_contents.replace(
98-
"keras.api._legacy", "keras.api._tf_keras.keras"
97+
"keras._legacy", "keras._tf_keras.keras"
9998
)
10099
if os.path.exists(core_api_fpath):
101100
with open(core_api_fpath) as f:
102101
core_api_contents = f.read()
103102
core_api_contents = core_api_contents.replace(
104-
"from keras.api import _tf_keras\n", ""
103+
"from keras import _tf_keras as _tf_keras\n", ""
105104
)
106105
for legacy_submodule in legacy_submodules:
107106
core_api_contents = core_api_contents.replace(
108-
f"from keras.api import {legacy_submodule}\n",
107+
f"from keras import {legacy_submodule} as {legacy_submodule}\n", # noqa: E501
109108
"",
110109
)
111110
core_api_contents = core_api_contents.replace(
112-
f"keras.api.{legacy_submodule}",
113-
f"keras.api._tf_keras.keras.{legacy_submodule}",
111+
f"keras.{legacy_submodule}",
112+
f"keras._tf_keras.keras.{legacy_submodule}",
114113
)
115114
# Remove duplicate generated comments string.
116115
legacy_contents = re.sub(r"\n", r"\\n", legacy_contents)
@@ -122,7 +121,7 @@ def create_legacy_directory(package_dir):
122121
)
123122
for import_name in legacy_imports:
124123
core_api_contents = re.sub(
125-
f"\n.* import {import_name}\n",
124+
f"\n.* import {import_name} as {import_name}\n",
126125
r"\n",
127126
core_api_contents,
128127
)
@@ -131,68 +130,41 @@ def create_legacy_directory(package_dir):
131130
f.write(legacy_contents)
132131

133132
# Delete keras/api/_legacy/
134-
shutil.rmtree(os.path.join(api_dir, "_legacy"))
133+
shutil.rmtree(os.path.join(package_dir, "_legacy"))
135134

136135

137136
def export_version_string(api_init_fname):
138137
with open(api_init_fname) as f:
139138
contents = f.read()
140139
with open(api_init_fname, "w") as f:
141-
contents += "from keras.src.version import __version__\n"
140+
contents += "from keras.src.version import __version__ as __version__\n"
142141
f.write(contents)
143142

144143

145-
def update_package_init(template_fname, dest_fname, api_module):
146-
with open(template_fname) as template_file:
147-
with open(dest_fname, "w") as dest_file:
148-
for line in template_file:
149-
if "# DO NOT EDIT." in line:
150-
dest_file.write(line)
151-
# Import all public symbols from `api/` and `__version__`.
152-
for symbol in api_module.__dict__.keys():
153-
if symbol.startswith("_") and symbol != "__version__":
154-
continue
155-
dest_file.write(f"from keras.api import {symbol}\n")
156-
# Skip the previous autogenerated block.
157-
for line in template_file:
158-
if "# END DO NOT EDIT." in line:
159-
break
160-
dest_file.write(line)
161-
162-
163144
def build():
164-
# Backup the `keras/__init__.py` and restore it on error in api gen.
165145
root_path = os.path.dirname(os.path.abspath(__file__))
166146
code_api_dir = os.path.join(root_path, PACKAGE, "api")
167-
code_init_fname = os.path.join(root_path, PACKAGE, "__init__.py")
168147
# Create temp build dir
169148
build_dir = copy_source_to_build_directory(root_path)
170-
build_api_dir = os.path.join(build_dir, PACKAGE, "api")
171-
build_init_fname = os.path.join(build_dir, PACKAGE, "__init__.py")
149+
build_api_dir = os.path.join(build_dir, PACKAGE)
150+
build_src_dir = os.path.join(build_api_dir, "src")
172151
build_api_init_fname = os.path.join(build_api_dir, "__init__.py")
173152
try:
174153
os.chdir(build_dir)
175-
# Generates `keras/api` directory.
176-
if os.path.exists(build_api_dir):
177-
shutil.rmtree(build_api_dir)
178-
if os.path.exists(build_init_fname):
179-
os.remove(build_init_fname)
180-
os.makedirs(build_api_dir)
181-
namex.generate_api_files(
182-
"keras", code_directory="src", target_directory="api"
183-
)
154+
open(build_api_init_fname, "w").close()
155+
namex.generate_api_files("keras", code_directory="src")
184156
# Add __version__ to `api/`.
185157
export_version_string(build_api_init_fname)
186158
# Creates `_tf_keras` with full keras API
187159
create_legacy_directory(package_dir=os.path.join(build_dir, PACKAGE))
188-
# Update toplevel init with all `api/` imports.
189-
api_module = importlib.import_module(f"{BUILD_DIR_NAME}.keras.api")
190-
update_package_init(code_init_fname, build_init_fname, api_module)
191160
# Copy back the keras/api and keras/__init__.py from build directory
161+
if os.path.exists(build_src_dir):
162+
shutil.rmtree(build_src_dir)
192163
if os.path.exists(code_api_dir):
193164
shutil.rmtree(code_api_dir)
194-
shutil.copytree(build_api_dir, code_api_dir)
195-
shutil.copy(build_init_fname, code_init_fname)
165+
shutil.copytree(
166+
build_api_dir, code_api_dir, ignore=shutil.ignore_patterns("src/")
167+
)
196168
finally:
197169
# Clean up: remove the build directory (no longer needed)
198170
shutil.rmtree(build_dir)

‎keras/__init__.py‎

Lines changed: 5 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -1,80 +1,13 @@
1-
# DO NOT EDIT. Generated by api_gen.sh
2-
from keras.api import DTypePolicy
3-
from keras.api import FloatDTypePolicy
4-
from keras.api import Function
5-
from keras.api import Initializer
6-
from keras.api import Input
7-
from keras.api import InputSpec
8-
from keras.api import KerasTensor
9-
from keras.api import Layer
10-
from keras.api import Loss
11-
from keras.api import Metric
12-
from keras.api import Model
13-
from keras.api import Operation
14-
from keras.api import Optimizer
15-
from keras.api import Quantizer
16-
from keras.api import Regularizer
17-
from keras.api import RematScope
18-
from keras.api import Sequential
19-
from keras.api import StatelessScope
20-
from keras.api import SymbolicScope
21-
from keras.api import Variable
22-
from keras.api import __version__
23-
from keras.api import activations
24-
from keras.api import applications
25-
from keras.api import backend
26-
from keras.api import callbacks
27-
from keras.api import config
28-
from keras.api import constraints
29-
from keras.api import datasets
30-
from keras.api import device
31-
from keras.api import distribution
32-
from keras.api import dtype_policies
33-
from keras.api import export
34-
from keras.api import initializers
35-
from keras.api import layers
36-
from keras.api import legacy
37-
from keras.api import losses
38-
from keras.api import metrics
39-
from keras.api import mixed_precision
40-
from keras.api import models
41-
from keras.api import name_scope
42-
from keras.api import ops
43-
from keras.api import optimizers
44-
from keras.api import preprocessing
45-
from keras.api import quantizers
46-
from keras.api import random
47-
from keras.api import regularizers
48-
from keras.api import remat
49-
from keras.api import saving
50-
from keras.api import tree
51-
from keras.api import utils
52-
from keras.api import version
53-
from keras.api import visualization
54-
from keras.api import wrappers
55-
56-
# END DO NOT EDIT.
1+
# This file should NEVER be packaged! This is a hack to make "import keras" from
2+
# the base of the repo just import the source files. We'll keep it for compat.
573

584
import os # isort: skip
595

606
# Add everything in /api/ to the module search path.
617
__path__.append(os.path.join(os.path.dirname(__file__), "api")) # noqa: F405
628

9+
from keras.api import * # noqa: F403, E402
10+
from keras.api import __version__ # noqa: E402
11+
6312
# Don't pollute namespace.
6413
del os
65-
66-
67-
# Never autocomplete `.src` or `.api` on an imported keras object.
68-
def __dir__():
69-
keys = dict.fromkeys((globals().keys()))
70-
keys.pop("src")
71-
keys.pop("api")
72-
return list(keys)
73-
74-
75-
# Don't import `.src` or `.api` during `from keras import *`.
76-
__all__ = [
77-
name
78-
for name in globals().keys()
79-
if not (name.startswith("_") or name in ("src", "api"))
80-
]

‎keras/api/__init__.py‎

Lines changed: 60 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -4,57 +4,63 @@
44
since your modifications would be overwritten.
55
"""
66

7-
from keras.api import _tf_keras
8-
from keras.api import activations
9-
from keras.api import applications
10-
from keras.api import backend
11-
from keras.api import callbacks
12-
from keras.api import config
13-
from keras.api import constraints
14-
from keras.api import datasets
15-
from keras.api import distribution
16-
from keras.api import dtype_policies
17-
from keras.api import export
18-
from keras.api import initializers
19-
from keras.api import layers
20-
from keras.api import legacy
21-
from keras.api import losses
22-
from keras.api import metrics
23-
from keras.api import mixed_precision
24-
from keras.api import models
25-
from keras.api import ops
26-
from keras.api import optimizers
27-
from keras.api import preprocessing
28-
from keras.api import quantizers
29-
from keras.api import random
30-
from keras.api import regularizers
31-
from keras.api import saving
32-
from keras.api import tree
33-
from keras.api import utils
34-
from keras.api import visualization
35-
from keras.api import wrappers
36-
from keras.src.backend import Variable
37-
from keras.src.backend import device
38-
from keras.src.backend import name_scope
39-
from keras.src.backend.common.keras_tensor import KerasTensor
40-
from keras.src.backend.common.remat import RematScope
41-
from keras.src.backend.common.remat import remat
42-
from keras.src.backend.common.stateless_scope import StatelessScope
43-
from keras.src.backend.common.symbolic_scope import SymbolicScope
44-
from keras.src.dtype_policies.dtype_policy import DTypePolicy
45-
from keras.src.dtype_policies.dtype_policy import FloatDTypePolicy
46-
from keras.src.initializers.initializer import Initializer
47-
from keras.src.layers.core.input_layer import Input
48-
from keras.src.layers.input_spec import InputSpec
49-
from keras.src.layers.layer import Layer
50-
from keras.src.losses.loss import Loss
51-
from keras.src.metrics.metric import Metric
52-
from keras.src.models.model import Model
53-
from keras.src.models.sequential import Sequential
54-
from keras.src.ops.function import Function
55-
from keras.src.ops.operation import Operation
56-
from keras.src.optimizers.optimizer import Optimizer
57-
from keras.src.quantizers.quantizers import Quantizer
58-
from keras.src.regularizers.regularizers import Regularizer
59-
from keras.src.version import __version__
60-
from keras.src.version import version
7+
from keras import _tf_keras as _tf_keras
8+
from keras import activations as activations
9+
from keras import applications as applications
10+
from keras import backend as backend
11+
from keras import callbacks as callbacks
12+
from keras import config as config
13+
from keras import constraints as constraints
14+
from keras import datasets as datasets
15+
from keras import distribution as distribution
16+
from keras import dtype_policies as dtype_policies
17+
from keras import export as export
18+
from keras import initializers as initializers
19+
from keras import layers as layers
20+
from keras import legacy as legacy
21+
from keras import losses as losses
22+
from keras import metrics as metrics
23+
from keras import mixed_precision as mixed_precision
24+
from keras import models as models
25+
from keras import ops as ops
26+
from keras import optimizers as optimizers
27+
from keras import preprocessing as preprocessing
28+
from keras import quantizers as quantizers
29+
from keras import random as random
30+
from keras import regularizers as regularizers
31+
from keras import saving as saving
32+
from keras import tree as tree
33+
from keras import utils as utils
34+
from keras import visualization as visualization
35+
from keras import wrappers as wrappers
36+
from keras.src.backend import Variable as Variable
37+
from keras.src.backend import device as device
38+
from keras.src.backend import name_scope as name_scope
39+
from keras.src.backend.common.keras_tensor import KerasTensor as KerasTensor
40+
from keras.src.backend.common.remat import RematScope as RematScope
41+
from keras.src.backend.common.remat import remat as remat
42+
from keras.src.backend.common.stateless_scope import (
43+
StatelessScope as StatelessScope,
44+
)
45+
from keras.src.backend.common.symbolic_scope import (
46+
SymbolicScope as SymbolicScope,
47+
)
48+
from keras.src.dtype_policies.dtype_policy import DTypePolicy as DTypePolicy
49+
from keras.src.dtype_policies.dtype_policy import (
50+
FloatDTypePolicy as FloatDTypePolicy,
51+
)
52+
from keras.src.initializers.initializer import Initializer as Initializer
53+
from keras.src.layers.core.input_layer import Input as Input
54+
from keras.src.layers.input_spec import InputSpec as InputSpec
55+
from keras.src.layers.layer import Layer as Layer
56+
from keras.src.losses.loss import Loss as Loss
57+
from keras.src.metrics.metric import Metric as Metric
58+
from keras.src.models.model import Model as Model
59+
from keras.src.models.sequential import Sequential as Sequential
60+
from keras.src.ops.function import Function as Function
61+
from keras.src.ops.operation import Operation as Operation
62+
from keras.src.optimizers.optimizer import Optimizer as Optimizer
63+
from keras.src.quantizers.quantizers import Quantizer as Quantizer
64+
from keras.src.regularizers.regularizers import Regularizer as Regularizer
65+
from keras.src.version import __version__ as __version__
66+
from keras.src.version import version as version

0 commit comments

Comments
 (0)