Merge remote-tracking branch 'origin/main' into jakopina-view2

llvm · Sep 27, 2022 · 7a9669b · 7a9669b
2 parents 0a733f4 + b0b2b3a
commit 7a9669b
Show file tree

Hide file tree

Showing 67 changed files with 10,299 additions and 8,806 deletions.
diff --git a/.github/workflows/buildAndTest.yml b/.github/workflows/buildAndTest.yml
@@ -17,8 +17,8 @@ concurrency:
 
 
 # Provisioned Jobs:
-# ubuntu - x86_64 - llvm in-tree     - pytorch binary - build+test    # most used dev flow and fastest signal
-# ubuntu - x86_64 - llvm out-of-tree - pytorch source - build+test    # most elaborate build
+# ubuntu/docker - x86_64 - llvm in-tree     - pytorch binary - build+test    # most used dev flow and fastest signal
+# ubuntu/docker - x86_64 - llvm out-of-tree - pytorch source - build+test    # most elaborate build
 # macos  - arm64  - llvm in-tree     - pytorch binary - build only    # cross compile, can't test arm64
 jobs:
   build-test:
@@ -57,60 +57,11 @@ jobs:
       with:
         cache-suffix: ${{ matrix.os-arch }}-${{ matrix.llvm-build }}-${{ matrix.torch-binary }}
 
-    - name: Configure os-arch='ubuntu-x86_64' llvm-build='in-tree' torch-binary='${{ matrix.torch-binary }}'
-      # Fastest build, most used dev flow
-      if: ${{ matrix.os-arch == 'ubuntu-x86_64' && matrix.llvm-build == 'in-tree' }}
-      run: |
-        cmake -GNinja -Bbuild \
-          -DCMAKE_BUILD_TYPE=Release \
-          -DCMAKE_C_COMPILER=clang \
-          -DCMAKE_CXX_COMPILER=clang++ \
-          -DCMAKE_C_COMPILER_LAUNCHER=ccache \
-          -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-          -DCMAKE_LINKER=lld \
-          -DLLVM_ENABLE_ASSERTIONS=ON \
-          -DLLVM_ENABLE_PROJECTS=mlir \
-          -DLLVM_EXTERNAL_PROJECTS="torch-mlir;torch-mlir-dialects" \
-          -DLLVM_EXTERNAL_TORCH_MLIR_SOURCE_DIR="$GITHUB_WORKSPACE" \
-          -DLLVM_EXTERNAL_TORCH_MLIR_DIALECTS_SOURCE_DIR="${GITHUB_WORKSPACE}/externals/llvm-external-projects/torch-mlir-dialects" \
-          -DLLVM_TARGETS_TO_BUILD=host \
-          -DMLIR_ENABLE_BINDINGS_PYTHON=ON \
-          -DTORCH_MLIR_USE_INSTALLED_PYTORCH="${{ matrix.torch-binary }}" \
-          -DPython3_EXECUTABLE="$(which python)" \
-          $GITHUB_WORKSPACE/externals/llvm-project/llvm
-
-    - name: Configure os-arch='ubuntu-x86_64' llvm-build='out-of-tree' torch-binary='${{ matrix.torch-binary }}'
-      # Most elaborate build, but cached
-      if: ${{ matrix.os-arch == 'ubuntu-x86_64' && matrix.llvm-build == 'out-of-tree' }}
+    - name: Build and Test os-arch='ubuntu-x86_64' llvm-build='${{ matrix.llvm-build }}' torch-binary='${{ matrix.torch-binary }}'
+      if: ${{ matrix.os-arch == 'ubuntu-x86_64' }}
       run: |
-        cmake -GNinja -Bllvm-build \
-          -DCMAKE_BUILD_TYPE=Release \
-          -DCMAKE_C_COMPILER=clang \
-          -DCMAKE_CXX_COMPILER=clang++ \
-          -DCMAKE_C_COMPILER_LAUNCHER=ccache \
-          -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-          -DCMAKE_LINKER=lld \
-          -DLLVM_ENABLE_ASSERTIONS=ON \
-          -DLLVM_ENABLE_PROJECTS=mlir \
-          -DLLVM_TARGETS_TO_BUILD=host \
-          -DMLIR_ENABLE_BINDINGS_PYTHON=ON \
-          -DPython3_EXECUTABLE="$(which python)" \
-          $GITHUB_WORKSPACE/externals/llvm-project/llvm
-        cmake --build llvm-build
-
-        cmake -GNinja -Bbuild \
-          -DCMAKE_C_COMPILER=clang \
-          -DCMAKE_CXX_COMPILER=clang++ \
-          -DCMAKE_C_COMPILER_LAUNCHER=ccache \
-          -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-          -DCMAKE_LINKER=lld \
-          -DLLVM_DIR="$GITHUB_WORKSPACE/llvm-build/lib/cmake/llvm/" \
-          -DMLIR_DIR="$GITHUB_WORKSPACE/llvm-build/lib/cmake/mlir/" \
-          -DMLIR_ENABLE_BINDINGS_PYTHON=OFF \
-          -DTORCH_MLIR_USE_INSTALLED_PYTORCH="${{ matrix.torch-binary }}" \
-          -DPython3_EXECUTABLE="$(which python)" \
-          $GITHUB_WORKSPACE
-
+        cd $GITHUB_WORKSPACE
+        TM_PACKAGES="${{ matrix.llvm-build }}" TM_USE_PYTORCH_BINARY="${{ matrix.torch-binary }}" ./build_tools/python_deploy/build_linux_packages.sh
     - name: Configure os-arch='macos-arm64' llvm-build='in-tree' torch-binary='${{ matrix.torch-binary }}'
       # cross compile, can't test arm64
       if: ${{ matrix.os-arch == 'macos-arm64' && matrix.llvm-build == 'in-tree' }}
@@ -139,63 +90,7 @@ jobs:
           -DMACOSX_DEPLOYMENT_TARGET=12.0 \
           -DPython3_EXECUTABLE="$(which python)" \
           $GITHUB_WORKSPACE/externals/llvm-project/llvm
-
-    - name: Build torch-mlir
-      if: ${{ matrix.os-arch == 'ubuntu-x86_64' }}
-      run: |
-        cmake --build build
-
     - name: Build torch-mlir (cross-compile)
       if: ${{ matrix.os-arch == 'macos-arm64' }}
       run: |
         cmake --build build_arm64
-
-    - name: Run torch-mlir unit tests
-      if: ${{ matrix.os-arch == 'ubuntu-x86_64' }}
-      run: |
-        cmake --build build --target check-torch-mlir-all
-
-    - name: Ensure generated files are up to date
-      if: ${{ matrix.os-arch == 'ubuntu-x86_64' && matrix.llvm-build == 'in-tree' }}
-      run: |
-        ./build_tools/update_torch_ods.sh
-        ./build_tools/update_shape_lib.sh
-        if ! git diff --quiet; then
-          echo "#######################################################"
-          echo "Generated files are not up to date (see diff below)"
-          echo ">>> Please run ./build_tools/update_torch_ods.sh and ./build_tools/update_shape_lib.sh <<<"
-          echo "#######################################################"
-          git diff --color=always
-          exit 1
-        fi
-
-    - name: Run refbackend e2e integration tests
-      if: ${{ matrix.os-arch == 'ubuntu-x86_64' && matrix.llvm-build == 'in-tree' }}
-      run: |
-        export PYTHONPATH="$GITHUB_WORKSPACE/build/tools/torch-mlir/python_packages/torch_mlir"
-        python -m e2e_testing.main --config=refbackend -v
-
-    - name: Run eager_mode e2e integration tests
-      if: ${{ matrix.os-arch == 'ubuntu-x86_64' && matrix.llvm-build == 'in-tree' }}
-      run: |
-        export PYTHONPATH="$GITHUB_WORKSPACE/build/tools/torch-mlir/python_packages/torch_mlir"
-        python -m e2e_testing.main --config=eager_mode -v
-
-    - name: Run mhlo e2e integration tests
-      if: ${{ matrix.os-arch == 'ubuntu-x86_64' && matrix.llvm-build == 'in-tree' }}
-      run: |
-        export PYTHONPATH="$GITHUB_WORKSPACE/build/tools/torch-mlir/python_packages/torch_mlir"
-        python -m e2e_testing.main --config=mhlo -v
-
-    - name: Run tosa e2e integration tests
-      if: ${{ matrix.os-arch == 'ubuntu-x86_64' && matrix.llvm-build == 'in-tree' }}
-      run: |
-        export PYTHONPATH="$GITHUB_WORKSPACE/build/tools/torch-mlir/python_packages/torch_mlir"
-        python -m e2e_testing.main --config=tosa -v
-
-    - name: Run lazy_tensor_core e2e integration tests
-      if: ${{ matrix.os-arch == 'ubuntu-x86_64' && matrix.llvm-build == 'in-tree' }}
-      run: |
-        export PYTHONPATH="$GITHUB_WORKSPACE/build/tools/torch-mlir/python_packages/torch_mlir"
-        echo "LTC tests disabled temporarily. https://github.com/llvm/torch-mlir/pull/1292"
-        # python -m e2e_testing.main --config=lazy_tensor_core -v
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -54,7 +54,9 @@ torch_mlir_add_llvm_external_project(
   TORCH_MLIR_DIALECTS
   ${CMAKE_CURRENT_SOURCE_DIR}/externals/llvm-external-projects/torch-mlir-dialects)
 
-if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
+option(TORCH_MLIR_OOT_BUILD "Specifies an out of tree build" OFF)
+
+if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR OR TORCH_MLIR_OOT_BUILD)
   message(STATUS "Torch-MLIR out-of-tree build.")
   # Out-of-tree build
 

diff --git a/README.md b/README.md
@@ -22,7 +22,7 @@ Multiple Vendors use MLIR as the middle layer, mapping from platform frameworks
 
 We have few paths to lower down to the Torch MLIR Dialect.
 
-![Torch Lowering Architectures](Torch-MLIR.png)
+![Torch Lowering Architectures](docs/Torch-MLIR.png)
 
  - TorchScript
     This is the most tested path down to Torch MLIR Dialect, and the PyTorch ecosystem is converging on using TorchScript IR as a lingua franca.

diff --git a/Torch-MLIR.png b/Torch-MLIR.png
diff --git a/build_tools/autogen_ltc_backend.py b/build_tools/autogen_ltc_backend.py
@@ -264,6 +264,9 @@ def get_opnames(ops):
         # Additional ops to support that are not supported by Torch-MLIR explicitly
         supported |= set(config.get("additional_ops", []))
 
+        # List of ops that will take in symints for its size
+        symint = set(config.get("symint", []))
+
         self.ops = sorted(ops)
 
         with self.source_yaml.open("w") as f:
@@ -272,6 +275,7 @@ def get_opnames(ops):
                 "cpp_namespace": "torch::lazy",
                 "full_codegen": self.ops,
                 "supported": sorted(supported),
+                "symint": sorted(symint),
                 "non_native": non_native,
             }
             yaml.dump(source_yaml, f, default_flow_style=False)

diff --git a/build_tools/autogen_ltc_backend.yaml b/build_tools/autogen_ltc_backend.yaml
@@ -16,6 +16,7 @@ blacklist:
 - copy_
 
 # Disabled for consistency with TS backend
+- lift_fresh_copy
 - new_empty
 - rsub
 - slice.Tensor  # Disabled in favour of slice_copy.Tensor
@@ -60,6 +61,7 @@ supported:
 # but their implementations call view operators (which we need to functionalize away).
 - block_diag
 - new_empty_strided
+- narrow_copy
 - pixel_shuffle
 - pixel_unshuffle
 - select_backward
@@ -69,6 +71,16 @@ supported:
 - linalg_pinv.atol_rtol_tensor
 - logsumexp.out
 
+# List of ops that will take in symints for the size instead of ints
+symint:
+- empty.memory_format
+- new_empty_strided
+- expand
+- expand_copy
+- narrow_copy
+- view
+- view_copy
+
 
 additional_ops:
 # Additional ops to support that are not supported by Torch-MLIR explicitly

diff --git a/build_tools/python_deploy/build_linux_packages.sh b/build_tools/python_deploy/build_linux_packages.sh
@@ -48,7 +48,7 @@ TM_PYTHON_VERSIONS="${TM_PYTHON_VERSIONS:-cp39-cp39 cp310-cp310}"
 # Location to store Release wheels
 TM_OUTPUT_DIR="${TM_OUTPUT_DIR:-${this_dir}/wheelhouse}"
 # What "packages to build"
-TM_PACKAGES="${TM_PACKAGES:-torch-mlir out-of-tree in-tree}"
+TM_PACKAGES="${TM_PACKAGES:-torch-mlir}"
 # Use pre-built Pytorch
 TM_USE_PYTORCH_BINARY="${TM_USE_PYTORCH_BINARY:-ON}"
 # Skip running tests if you want quick iteration
@@ -211,11 +211,9 @@ function _check_file_not_changed_by() {
     # TODO: Is there a better cleanup strategy that doesn't require duplicating
     # this inside and outside the `if`?
     rm "$file_new"
-    rm "$file_backup"
     return 1
   fi
   rm "$file_new"
-  rm "$file_backup"
 }
 
 function test_in_tree() {
@@ -238,7 +236,9 @@ function test_in_tree() {
   python -m e2e_testing.main --config=eager_mode -v
 
   echo ":::: Run TOSA e2e integration tests"
-  python -m e2e_testing.main --config=tosa -v
+  # crashing_tests_to_not_attempt_to_run_and_a_bug_is_filed issues:
+  # - AvgPool2dFloatModule_basic,AvgPool2dCeilModeTrueModule_basic: https://github.com/llvm/torch-mlir/issues/1361
+  python -m e2e_testing.main --config=tosa -v --crashing_tests_to_not_attempt_to_run_and_a_bug_is_filed AvgPool2dFloatModule_basic AvgPool2dCeilModeTrueModule_basic
 
   # Temporarily disabled in top of main (https://github.com/llvm/torch-mlir/pull/1292)
   #echo ":::: Run Lazy Tensor Core e2e integration tests"