Avoid editable install of py_matter_idl in build env (#39826)

* Avoid editable install of py_matter_idl in build env

The editable install creates a problem for the way the configure script creates
the build env, because it assumes that it can create a self-contained, reusable
build env that keeps working even if the original source directory is removed
or moved to a different location. This is no longer true when the build env
contains effectively a "symlink" into the source tree.

This change instead modifies codegen.py and codegen_paths.py to directly find
py_matter_idl relative to the script directory and import them from there,
without relying on the editable install. The python-path pip package is used to
do this (essentially it's just a small utility for resolving a path and
prepending it to sys.path, allowing the code in the script itself to remain
readable and straightforward.)

* Avoid empty path in error message

* Include py_matter_idl dependencies in requirements.build.txt

* Ensure CDPATH doesn't affect configure script

Author: ksperling-apple

scripts/codegen.py

@@ -13,6 +13,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# py_matter_idl may not be installed in the pigweed venv.
+# Reference it directly from the source tree.
+from python_path import PythonPath
+
+with PythonPath('py_matter_idl', relative_to=__file__):
+    from matter.idl.generators.path_resolution import expand_path_for_idl
+    from matter.idl.generators.registry import GENERATORS, CodeGenerator
+    from matter.idl.generators.storage import FileSystemGeneratorStorage, GeneratorStorage
+    from matter.idl.matter_idl_parser import CreateParser
+
 import logging
 import sys
 
@@ -24,11 +34,6 @@
 except ImportError:
     _has_coloredlogs = False
 
-from matter.idl.generators.path_resolution import expand_path_for_idl
-from matter.idl.generators.registry import GENERATORS, CodeGenerator
-from matter.idl.generators.storage import FileSystemGeneratorStorage, GeneratorStorage
-from matter.idl.matter_idl_parser import CreateParser
-
 
 class ListGeneratedFilesStorage(GeneratorStorage):
     """

scripts/codegen_paths.py

@@ -13,6 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# py_matter_idl may not be installed in the pigweed venv.
+# Reference it directly from the source tree.
+from python_path import PythonPath
+
+with PythonPath('py_matter_idl', relative_to=__file__):
+    from matter.idl.generators.path_resolution import expand_path_for_idl
+    from matter.idl.matter_idl_parser import CreateParser
+
 import logging
 
 import click
@@ -23,9 +31,6 @@
 except ImportError:
     _has_coloredlogs = False
 
-from matter.idl.generators.path_resolution import expand_path_for_idl
-from matter.idl.matter_idl_parser import CreateParser
-
 # Supported log levels, mapping string values required for argument
 # parsing into logging constants
 __LOG_LEVELS__ = {

scripts/configure

@@ -36,6 +36,7 @@
 
 set -o pipefail
 shopt -s extglob
+unset CDPATH
 
 function usage() { # status
     info "Usage: $0 [OPTIONS] [--project=... [PROJECT OPTIONS]]"
@@ -67,7 +68,7 @@ function usage() { # status
 }
 
 function main() { # ...
-    CHIP_ROOT=$(cd "$(dirname "$0")/.." >/dev/null && pwd)
+    CHIP_ROOT=$(cd "$(dirname "$0")/.." && pwd)
     BUILD_ENV_DEPS=(
         "${CHIP_ROOT}/scripts/setup/requirements.build.txt"
         "${CHIP_ROOT}/scripts/setup/constraints.txt"
@@ -99,8 +100,8 @@ function main() { # ...
     [[ -n "$PROJECT" || -n "$BUILD_ENV_DIR" ]] || usage 1
 
     if [[ -n "$PROJECT" ]]; then
-        PROJECT_PATH="$(cd "${CHIP_ROOT}" >/dev/null && cd "${PROJECT}" >/dev/null && pwd)"
-        [[ -n "$PROJECT_PATH" && -r "${PROJECT_PATH}/.gn" ]] || fail "Invalid project '${PROJECT}' - missing .gn at '${PROJECT_PATH}'"
+        PROJECT_PATH="$(cd "${CHIP_ROOT}" 2>/dev/null && cd "${PROJECT}" 2>/dev/null && pwd)"
+        [[ -n "$PROJECT_PATH" && -r "${PROJECT_PATH}/.gn" ]] || fail "Invalid project '${PROJECT}' (no such directory or missing .gn file)"
     fi
 
     if [[ -n "$PW_ROOT" ]]; then
@@ -131,7 +132,7 @@ function main() { # ...
 
     if [[ -n "$BUILD_ENV_DIR" ]]; then
         mkdir -p "$BUILD_ENV_DIR"
-        BUILD_ENV_PATH="$(cd "$BUILD_ENV_DIR" >/dev/null && pwd)"
+        BUILD_ENV_PATH="$(cd "$BUILD_ENV_DIR" && pwd)"
         [[ -n "$BUILD_ENV_PATH" ]] || fail "Invalid build-env-dir '${BUILD_ENV_DIR}'"
         BUILD_ENV_DIR="$BUILD_ENV_PATH" # absolute
     else
@@ -281,7 +282,6 @@ function configure_python_env() {
     # Ensure pip and wheel are up to date first (using pip.pyz if necessary)
     "${BUILD_ENV_PATH}/bin/python3" "$pip" install --require-virtualenv --quiet --upgrade pip wheel
 
-    export PW_PROJECT_ROOT=$CHIP_ROOT
     "${BUILD_ENV_PATH}/bin/pip" install --require-virtualenv --quiet \
         -r "${CHIP_ROOT}/scripts/setup/requirements.build.txt" \
         -c "${CHIP_ROOT}/scripts/setup/constraints.txt"

scripts/setup/requirements.all.txt

@@ -19,6 +19,9 @@ pyobjc-core; sys_platform == 'darwin'
 pyobjc-framework-cocoa; sys_platform == 'darwin'
 pyobjc-framework-corebluetooth; sys_platform == 'darwin'
 
+# other tools
+click-option-group
+
 # python unit tests run directly without installing
 # built venv
 #

scripts/setup/requirements.build.txt

@@ -5,8 +5,9 @@
 
 # Ideally, core build scripts should depend only on the standard library.
 
-# scripts/build
+# scripts/build, scripts/codegen, py_matter_idl
 click
-click-option-group
-
--e ${PW_PROJECT_ROOT}/scripts/py_matter_idl
+coloredlogs
+jinja2
+lark
+python-path