mlx - numpy ops and merge

.github/workflows/actions.yml

@@ -136,7 +136,5 @@ jobs:
           pip install -r requirements-openvino.txt --progress-bar off --upgrade
           pip uninstall -y keras keras-nightly
           pip install -e "." --progress-bar off --upgrade
-      - name: Install pre-commit
-        run: pip install pre-commit && pre-commit install
       - name: Run pre-commit
         run: pre-commit run --all-files --hook-stage manual

.github/workflows/nightly.yml

@@ -82,26 +82,8 @@ jobs:
           pip install -r requirements-openvino.txt --progress-bar off --upgrade
           pip uninstall -y keras keras-nightly
           pip install -e "." --progress-bar off --upgrade
-      - name: Check for API changes
-        run: |
-          bash shell/api_gen.sh
-          git status
-          clean=$(git status | grep "nothing to commit")
-          if [ -z "$clean" ]; then
-            echo "Please run shell/api_gen.sh to generate API."
-            exit 1
-          fi
-      - name: Lint
-        run: bash shell/lint.sh
-      - name: Check for API changes
-        run: |
-          bash shell/api_gen.sh
-          git status
-          clean=$(git status | grep "nothing to commit")
-          if [ -z "$clean" ]; then
-            echo "Please run shell/api_gen.sh to generate API."
-            exit 1
-          fi
+      - name: Run pre-commit
+        run: pre-commit run --all-files --hook-stage manual
 
 
   nightly:

.github/workflows/scorecard.yml

@@ -48,14 +48,14 @@ jobs:
       # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
       # format to the repository Actions tab.
       - name: "Upload artifact"
-        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         with:
           name: SARIF file
           path: results.sarif
           retention-days: 5
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@b56ba49b26e50535fa1e7f7db0f4f7b4bf65d80d # v3.28.10
+        uses: github/codeql-action/upload-sarif@28deaeda66b76a05916b6923827895f2b14ab387 # v3.28.16
         with:
           sarif_file: results.sarif

.gitignore

@@ -13,7 +13,7 @@ obj/**
 tmp/**
 .vs/
 dist/**
-*.egg-info/*
+**/*.egg-info/*
 .vscode
 examples/**/*.jpg
 .python-version

README.md

@@ -93,7 +93,6 @@ the package has been imported.
 
 **Note:** The OpenVINO backend is an inference-only backend, meaning it is designed only for running model
 predictions using `model.predict()` method.
-To use `openvino` backend, install the required dependencies from the `requirements-openvino.txt` file.
 
 ## Backwards compatibility
 

api_gen.py

@@ -6,7 +6,6 @@
 It generates API and formats user and generated APIs.
 """
 
-import importlib
 import os
 import re
 import shutil
@@ -24,38 +23,38 @@ def ignore_files(_, filenames):
 def copy_source_to_build_directory(root_path):
     # Copy sources (`keras/` directory and setup files) to build dir
     build_dir = os.path.join(root_path, BUILD_DIR_NAME)
+    build_package_dir = os.path.join(build_dir, PACKAGE)
+    build_src_dir = os.path.join(build_package_dir, "src")
+    root_src_dir = os.path.join(root_path, PACKAGE, "src")
     if os.path.exists(build_dir):
         shutil.rmtree(build_dir)
-    os.mkdir(build_dir)
-    shutil.copytree(
-        PACKAGE, os.path.join(build_dir, PACKAGE), ignore=ignore_files
-    )
+    os.makedirs(build_package_dir)
+    shutil.copytree(root_src_dir, build_src_dir)
     return build_dir
 
 
 def create_legacy_directory(package_dir):
     src_dir = os.path.join(package_dir, "src")
-    api_dir = os.path.join(package_dir, "api")
     # Make keras/_tf_keras/ by copying keras/
-    tf_keras_dirpath_parent = os.path.join(api_dir, "_tf_keras")
+    tf_keras_dirpath_parent = os.path.join(package_dir, "_tf_keras")
     tf_keras_dirpath = os.path.join(tf_keras_dirpath_parent, "keras")
     os.makedirs(tf_keras_dirpath, exist_ok=True)
     with open(os.path.join(tf_keras_dirpath_parent, "__init__.py"), "w") as f:
-        f.write("from keras.api._tf_keras import keras\n")
-    with open(os.path.join(api_dir, "__init__.py")) as f:
+        f.write("from keras._tf_keras import keras\n")
+    with open(os.path.join(package_dir, "__init__.py")) as f:
         init_file = f.read()
         init_file = init_file.replace(
-            "from keras.api import _legacy",
-            "from keras.api import _tf_keras",
+            "from keras import _legacy as _legacy",
+            "from keras import _tf_keras as _tf_keras",
         )
-    with open(os.path.join(api_dir, "__init__.py"), "w") as f:
+    with open(os.path.join(package_dir, "__init__.py"), "w") as f:
         f.write(init_file)
     # Remove the import of `_tf_keras` in `keras/_tf_keras/keras/__init__.py`
-    init_file = init_file.replace("from keras.api import _tf_keras\n", "\n")
+    init_file = init_file.replace("from keras import _tf_keras\n", "\n")
     with open(os.path.join(tf_keras_dirpath, "__init__.py"), "w") as f:
         f.write(init_file)
-    for dirname in os.listdir(api_dir):
-        dirpath = os.path.join(api_dir, dirname)
+    for dirname in os.listdir(package_dir):
+        dirpath = os.path.join(package_dir, dirname)
         if os.path.isdir(dirpath) and dirname not in (
             "_legacy",
             "_tf_keras",
@@ -81,7 +80,7 @@ def create_legacy_directory(package_dir):
         for path in os.listdir(os.path.join(src_dir, "legacy"))
         if os.path.isdir(os.path.join(src_dir, "legacy", path))
     ]
-    for root, _, fnames in os.walk(os.path.join(api_dir, "_legacy")):
+    for root, _, fnames in os.walk(os.path.join(package_dir, "_legacy")):
         for fname in fnames:
             if fname.endswith(".py"):
                 legacy_fpath = os.path.join(root, fname)
@@ -95,22 +94,22 @@ def create_legacy_directory(package_dir):
                 with open(legacy_fpath) as f:
                     legacy_contents = f.read()
                     legacy_contents = legacy_contents.replace(
-                        "keras.api._legacy", "keras.api._tf_keras.keras"
+                        "keras._legacy", "keras._tf_keras.keras"
                     )
                 if os.path.exists(core_api_fpath):
                     with open(core_api_fpath) as f:
                         core_api_contents = f.read()
                     core_api_contents = core_api_contents.replace(
-                        "from keras.api import _tf_keras\n", ""
+                        "from keras import _tf_keras as _tf_keras\n", ""
                     )
                     for legacy_submodule in legacy_submodules:
                         core_api_contents = core_api_contents.replace(
-                            f"from keras.api import {legacy_submodule}\n",
+                            f"from keras import {legacy_submodule} as {legacy_submodule}\n",  # noqa: E501
                             "",
                         )
                         core_api_contents = core_api_contents.replace(
-                            f"keras.api.{legacy_submodule}",
-                            f"keras.api._tf_keras.keras.{legacy_submodule}",
+                            f"keras.{legacy_submodule}",
+                            f"keras._tf_keras.keras.{legacy_submodule}",
                         )
                     # Remove duplicate generated comments string.
                     legacy_contents = re.sub(r"\n", r"\\n", legacy_contents)
@@ -122,7 +121,7 @@ def create_legacy_directory(package_dir):
                     )
                     for import_name in legacy_imports:
                         core_api_contents = re.sub(
-                            f"\n.* import {import_name}\n",
+                            f"\n.* import {import_name} as {import_name}\n",
                             r"\n",
                             core_api_contents,
                         )
@@ -131,68 +130,41 @@ def create_legacy_directory(package_dir):
                     f.write(legacy_contents)
 
     # Delete keras/api/_legacy/
-    shutil.rmtree(os.path.join(api_dir, "_legacy"))
+    shutil.rmtree(os.path.join(package_dir, "_legacy"))
 
 
 def export_version_string(api_init_fname):
     with open(api_init_fname) as f:
         contents = f.read()
     with open(api_init_fname, "w") as f:
-        contents += "from keras.src.version import __version__\n"
+        contents += "from keras.src.version import __version__ as __version__\n"
         f.write(contents)
 
 
-def update_package_init(template_fname, dest_fname, api_module):
-    with open(template_fname) as template_file:
-        with open(dest_fname, "w") as dest_file:
-            for line in template_file:
-                if "# DO NOT EDIT." in line:
-                    dest_file.write(line)
-                    # Import all public symbols from `api/` and `__version__`.
-                    for symbol in api_module.__dict__.keys():
-                        if symbol.startswith("_") and symbol != "__version__":
-                            continue
-                        dest_file.write(f"from keras.api import {symbol}\n")
-                    # Skip the previous autogenerated block.
-                    for line in template_file:
-                        if "# END DO NOT EDIT." in line:
-                            break
-                dest_file.write(line)
-
-
 def build():
-    # Backup the `keras/__init__.py` and restore it on error in api gen.
     root_path = os.path.dirname(os.path.abspath(__file__))
     code_api_dir = os.path.join(root_path, PACKAGE, "api")
-    code_init_fname = os.path.join(root_path, PACKAGE, "__init__.py")
     # Create temp build dir
     build_dir = copy_source_to_build_directory(root_path)
-    build_api_dir = os.path.join(build_dir, PACKAGE, "api")
-    build_init_fname = os.path.join(build_dir, PACKAGE, "__init__.py")
+    build_api_dir = os.path.join(build_dir, PACKAGE)
+    build_src_dir = os.path.join(build_api_dir, "src")
     build_api_init_fname = os.path.join(build_api_dir, "__init__.py")
     try:
         os.chdir(build_dir)
-        # Generates `keras/api` directory.
-        if os.path.exists(build_api_dir):
-            shutil.rmtree(build_api_dir)
-        if os.path.exists(build_init_fname):
-            os.remove(build_init_fname)
-        os.makedirs(build_api_dir)
-        namex.generate_api_files(
-            "keras", code_directory="src", target_directory="api"
-        )
+        open(build_api_init_fname, "w").close()
+        namex.generate_api_files("keras", code_directory="src")
         # Add __version__ to `api/`.
         export_version_string(build_api_init_fname)
         # Creates `_tf_keras` with full keras API
         create_legacy_directory(package_dir=os.path.join(build_dir, PACKAGE))
-        # Update toplevel init with all `api/` imports.
-        api_module = importlib.import_module(f"{BUILD_DIR_NAME}.keras.api")
-        update_package_init(code_init_fname, build_init_fname, api_module)
         # Copy back the keras/api and keras/__init__.py from build directory
+        if os.path.exists(build_src_dir):
+            shutil.rmtree(build_src_dir)
         if os.path.exists(code_api_dir):
             shutil.rmtree(code_api_dir)
-        shutil.copytree(build_api_dir, code_api_dir)
-        shutil.copy(build_init_fname, code_init_fname)
+        shutil.copytree(
+            build_api_dir, code_api_dir, ignore=shutil.ignore_patterns("src/")
+        )
     finally:
         # Clean up: remove the build directory (no longer needed)
         shutil.rmtree(build_dir)

conftest.py

@@ -38,7 +38,7 @@ def pytest_collection_modifyitems(config, items):
             ]
 
     requires_trainable_backend = pytest.mark.skipif(
-        backend() == "numpy" or backend() == "openvino",
+        backend() in ["numpy", "openvino"],
         reason="Trainer not implemented for NumPy and OpenVINO backend.",
     )
     for item in items:

guides/training_with_built_in_methods.py

@@ -620,8 +620,8 @@ def __getitem__(self, idx):
 """
 To fit the model, pass the dataset instead as the `x` argument (no need for a `y`
 argument since the dataset includes the targets), and pass the validation dataset
-as the `validation_data` argument. And no need for the `batch_size` argument, since
-the dataset is already batched!
+as the `validation_data` argument. And no need for the `validation_batch_size`
+argument, since the dataset is already batched!
 """
 
 model = get_compiled_model()