Merge branch 'main' into features/#1117-array-copy-None

.github/pytorch-release-versions/pytorch-latest.txt

@@ -1 +1 @@
-1.13.1
+2.0.1

.github/release-drafter.yml

@@ -46,6 +46,8 @@ categories:
     label: 'graph'
   - title: 'Indexing'
     label: 'indexing'
+  - title: 'Interoperability'
+    label: 'interoperability'
   - title: 'Linear Algebra'
     label: 'linalg'
   - title: 'Logical'

.github/workflows/CIBase.yml

@@ -5,6 +5,7 @@ on:
     branches:
       - 'main'
       - 'release/**'
+      - 'support/**'
 
 jobs:
   starter:
@@ -15,6 +16,6 @@ jobs:
          curl -s -X POST \
            --fail \
            -F token=${{ secrets.CB_PIPELINE }} \
-           -F "ref=nvidiapytorch2211" \
+           -F "ref=pt20" \
            -F "variables[SHA]=$GITHUB_SHA" \
            https://codebase.helmholtz.cloud/api/v4/projects/7605/trigger/pipeline -o /dev/null

.github/workflows/CommentPR.yml

@@ -66,6 +66,6 @@ jobs:
          curl -s -X POST \
            --fail \
            -F token=${{ secrets.CB_PIPELINE }} \
-           -F "ref=nvidiapytorch2211" \
+           -F "ref=pt20" \
            -F "variables[PR]=${{ needs.upload.outputs.PR_NR }}" \
            https://codebase.helmholtz.cloud/api/v4/projects/7605/trigger/pipeline -o /dev/null

.github/workflows/ReceivePR.yml

@@ -7,6 +7,7 @@ on:
 
 jobs:
   build:
+    if: ${{ !startsWith(github.head_ref, 'support/') }}
     runs-on: ubuntu-latest
 
     steps:

.github/workflows/ci.yaml

@@ -12,21 +12,26 @@ jobs:
       fail-fast: false
       matrix:
         py-version:
-          - 3.7
           - 3.8
           - 3.9
+          - '3.10'
         mpi: [ 'openmpi' ]
         install-options: [ '.', '.[hdf5,netcdf]' ]
         pytorch-version:
-          - 'torch==1.7.1+cpu torchvision==0.8.2+cpu torchaudio==0.7.2'
           - 'torch==1.8.1+cpu torchvision==0.9.1+cpu torchaudio==0.8.1'
           - 'torch==1.9.1+cpu torchvision==0.10.1+cpu torchaudio==0.9.1'
           - 'torch==1.10.1+cpu torchvision==0.11.2+cpu torchaudio==0.10.1'
           - 'torch==1.11.0+cpu torchvision==0.12.0+cpu torchaudio==0.11.0'
           - 'torch==1.12.1+cpu torchvision==0.13.1+cpu torchaudio==0.12.1'
+          - 'torch==1.13.1+cpu torchvision==0.14.1+cpu torchaudio==0.13.1'
+          - 'torch torchvision torchaudio'
         exclude:
-          - py-version: 3.9
-            pytorch-version: 'torch==1.7.1+cpu torchvision==0.8.2+cpu torchaudio==0.7.2'
+          - py-version: '3.10'
+            pytorch-version: 'torch==1.8.1+cpu torchvision==0.9.1+cpu torchaudio==0.8.1'
+          - py-version: '3.10'
+            pytorch-version: 'torch==1.9.1+cpu torchvision==0.10.1+cpu torchaudio==0.9.1'
+          - py-version: '3.10'
+            pytorch-version: 'torch==1.10.1+cpu torchvision==0.11.2+cpu torchaudio==0.10.1'
 
 
     name: Python ${{ matrix.py-version }} with ${{ matrix.pytorch-version }}; options ${{ matrix.install-options }}
@@ -45,7 +50,7 @@ jobs:
       - name: Test
         run: |
           pip install pytest
-          pip install ${{ matrix.pytorch-version }} -f https://download.pytorch.org/whl/torch_stable.html
+          pip install ${{ matrix.pytorch-version }} --extra-index-url https://download.pytorch.org/whl/cpu
           pip install ${{ matrix.install-options }}
           mpirun -n 3 pytest heat/
           mpirun -n 4 pytest heat/

.github/workflows/latest-pytorch-support.yml

@@ -3,8 +3,8 @@ name: Support latest PyTorch
 on:
   push:
     branches:
-      - 'workflows/new-pytorch-release-branch'
-      - 'workflows/new-pytorch-main-branch'
+      - 'support/new-pytorch-release-branch'
+      - 'support/new-pytorch-main-branch'
     paths:
       - '.github/pytorch-release-versions/*'
 env:
@@ -55,6 +55,7 @@ jobs:
         uses: peter-evans/create-pull-request@v4
         with:
             base: ${{ env.base_branch }}
+            branch: ${{ env.working_branch }}
             delete-branch: true
             token: ${{ secrets.GITHUB_TOKEN }}
             commit-message: Support latest PyTorch release

.github/workflows/pytorch-latest-main.yml

@@ -2,7 +2,7 @@ name: Get latest PyTorch version main branch
 on:
   workflow_dispatch:
 env:
-  working_branch: workflows/new-pytorch-main-branch
+  working_branch: support/new-pytorch-main-branch
   base_branch: main
 permissions:
   contents: write

.github/workflows/pytorch-latest-release.yml

@@ -2,7 +2,7 @@ name: Get latest PyTorch version release branch
 on:
   workflow_dispatch:
 env:
-  working_branch: workflows/new-pytorch-release-branch
+  working_branch: support/new-pytorch-release-branch
   base_branch: release/1.2.x
 permissions:
   contents: write

heat/core/_operations.py

@@ -296,7 +296,11 @@ def __cum_op(
         return out
 
     return factories.array(
-        cumop, dtype=x.dtype if dtype is None else dtype, is_split=x.split, device=x.device
+        cumop,
+        dtype=x.dtype if dtype is None else dtype,
+        is_split=x.split,
+        device=x.device,
+        comm=x.comm,
     )
 
 

heat/core/arithmetics.py

@@ -131,7 +131,7 @@ def bitwise_and(t1: Union[DNDarray, float], t2: Union[DNDarray, float]) -> DNDar
         if heat_type_is_inexact(dt):
             raise TypeError("Operation is not supported for float types")
 
-    return _operations.__binary_op(torch.Tensor.__and__, t1, t2)
+    return _operations.__binary_op(torch.bitwise_and, t1, t2)
 
 
 DNDarray.__and__ = lambda self, other: bitwise_and(self, other)
@@ -175,7 +175,7 @@ def bitwise_or(t1: Union[DNDarray, float], t2: Union[DNDarray, float]) -> DNDarr
         if heat_type_is_inexact(dt):
             raise TypeError("Operation is not supported for float types")
 
-    return _operations.__binary_op(torch.Tensor.__or__, t1, t2)
+    return _operations.__binary_op(torch.bitwise_or, t1, t2)
 
 
 DNDarray.__or__ = lambda self, other: bitwise_or(self, other)
@@ -214,7 +214,7 @@ def bitwise_xor(t1: Union[DNDarray, float], t2: Union[DNDarray, float]) -> DNDar
         if heat_type_is_inexact(dt):
             raise TypeError("Operation is not supported for float types")
 
-    return _operations.__binary_op(torch.Tensor.__xor__, t1, t2)
+    return _operations.__binary_op(torch.bitwise_xor, t1, t2)
 
 
 DNDarray.__xor__ = lambda self, other: bitwise_xor(self, other)
@@ -531,10 +531,7 @@ def floordiv(t1: Union[DNDarray, float], t2: Union[DNDarray, float]) -> DNDarray
     DNDarray([[1., 0.],
               [1., 1.]], dtype=ht.float32, device=cpu:0, split=None)
     """
-    if int(torch.__version__.split(".")[1]) > 7:
-        return _operations.__binary_op(torch.div, t1, t2, fn_kwargs={"rounding_mode": "floor"})
-    else:
-        return _operations.__binary_op(torch.floor_divide, t1, t2)
+    return _operations.__binary_op(torch.div, t1, t2, fn_kwargs={"rounding_mode": "floor"})
 
 
 DNDarray.__floordiv__ = lambda self, other: floordiv(self, other)

heat/core/dndarray.py

@@ -463,6 +463,12 @@ def __cat_halo(self) -> torch.Tensor:
             dim=self.split,
         )
 
+    def __array__(self) -> np.ndarray:
+        """
+        Returns a view of the process-local slice of the :class:`DNDarray` as a numpy ndarray, if the ``DNDarray`` resides on CPU. Otherwise, it returns a copy, on CPU, of the process-local slice of ``DNDarray`` as numpy ndarray.
+        """
+        return self.larray.cpu().__array__()
+
     def astype(self, dtype, copy=True) -> DNDarray:
         """
         Returns a casted version of this array.
@@ -472,7 +478,7 @@ def astype(self, dtype, copy=True) -> DNDarray:
         Parameters
         ----------
         dtype : datatype
-            HeAT type to which the array is cast
+            Heat type to which the array is cast
         copy : bool, optional
             By default the operation returns a copy of this array. If copy is set to ``False`` the cast is performed
             in-place and this array is returned
@@ -1101,6 +1107,10 @@ def item(self):
         >>> x.item()
         0.0
         """
+        if self.size > 1:
+            raise ValueError("only one-element DNDarrays can be converted to Python scalars")
+        # make sure the element is on every process
+        self.resplit_(None)
         return self.__array.item()
 
     def __len__(self) -> int:
@@ -1111,8 +1121,9 @@ def __len__(self) -> int:
 
     def numpy(self) -> np.array:
         """
-        Convert :class:`DNDarray` to numpy array. If the ``DNDarray`` is distributed it will be merged beforehand. If the ``DNDarray``
-        resides on the GPU, it will be copied to the CPU first.
+        Returns a copy of the :class:`DNDarray` as numpy ndarray. If the ``DNDarray`` resides on the GPU, the underlying data will be copied to the CPU first.
+
+        If the ``DNDarray`` is distributed, an MPI Allgather operation will be performed before converting to np.ndarray, i.e. each MPI process will end up holding a copy of the entire array in memory.  Make sure process memory is sufficient!
 
         Examples
         --------
@@ -1609,7 +1620,7 @@ def __setitem__(
         for c, k in enumerate(key):
             try:
                 key[c] = k.item()
-            except (AttributeError, ValueError):
+            except (AttributeError, ValueError, RuntimeError):
                 pass
 
         rank = self.comm.rank

heat/core/linalg/basics.py

@@ -116,7 +116,7 @@ def cross(
         a_2d = True
         shape = tuple(1 if i == axisa else j for i, j in enumerate(a.shape))
         a = manipulations.concatenate(
-            [a, factories.zeros(shape, dtype=a.dtype, device=a.device)], axis=axisa
+            [a, factories.zeros(shape, dtype=a.dtype, device=a.device, comm=a.comm)], axis=axisa
         )
     if b.shape[axisb] == 2:
         b_2d = True
@@ -207,7 +207,7 @@ def det(a: DNDarray) -> DNDarray:
 
     acopy = a.copy()
     acopy = manipulations.reshape(acopy, (-1, m, m), new_split=a.split - a.ndim + 3)
-    adet = factories.ones(acopy.shape[0], dtype=a.dtype, device=a.device)
+    adet = factories.ones(acopy.shape[0], dtype=a.dtype, device=a.device, comm=a.comm)
 
     for k in range(adet.shape[0]):
         m = 0
@@ -475,6 +475,7 @@ def matmul(a: DNDarray, b: DNDarray, allow_resplit: bool = False) -> DNDarray:
     [1/1] tensor([[3., 1., 1., 1., 1., 1., 1.],
                   [4., 1., 1., 1., 1., 1., 1.]])
     >>> linalg.matmul(a, b).larray
+
     [0/1] tensor([[18.,  8.,  9., 10.],
                   [14.,  6.,  7.,  8.],
                   [18.,  7.,  8.,  9.],
@@ -520,7 +521,7 @@ def matmul(a: DNDarray, b: DNDarray, allow_resplit: bool = False) -> DNDarray:
     if a.split is None and b.split is None:  # matmul from torch
         if len(a.gshape) < 2 or len(b.gshape) < 2 or not allow_resplit:
             # if either of A or B is a vector
-            ret = factories.array(torch.matmul(a.larray, b.larray), device=a.device)
+            ret = factories.array(torch.matmul(a.larray, b.larray), device=a.device, comm=a.comm)
             if gpu_int_flag:
                 ret = og_type(ret, device=a.device)
             return ret
@@ -529,7 +530,7 @@ def matmul(a: DNDarray, b: DNDarray, allow_resplit: bool = False) -> DNDarray:
         slice_0 = a.comm.chunk(a.shape, a.split)[2][0]
         hold = a.larray @ b.larray
 
-        c = factories.zeros((a.gshape[-2], b.gshape[1]), dtype=c_type, device=a.device)
+        c = factories.zeros((a.gshape[-2], b.gshape[1]), dtype=c_type, device=a.device, comm=a.comm)
         c.larray[slice_0.start : slice_0.stop, :] += hold
         c.comm.Allreduce(MPI.IN_PLACE, c, MPI.SUM)
         if gpu_int_flag:
@@ -544,7 +545,7 @@ def matmul(a: DNDarray, b: DNDarray, allow_resplit: bool = False) -> DNDarray:
         b.resplit_(0)
         res = a.larray @ b.larray
         a.comm.Allreduce(MPI.IN_PLACE, res, MPI.SUM)
-        ret = factories.array(res, split=None, device=a.device)
+        ret = factories.array(res, split=None, device=a.device, comm=a.comm)
         if gpu_int_flag:
             ret = og_type(ret, device=a.device)
         return ret
@@ -567,7 +568,9 @@ def matmul(a: DNDarray, b: DNDarray, allow_resplit: bool = False) -> DNDarray:
     ) and not vector_flag:
         split = a.split if a.split is not None else b.split
         split = split if not vector_flag else 0
-        c = factories.zeros((a.gshape[-2], b.gshape[1]), split=split, dtype=c_type, device=a.device)
+        c = factories.zeros(
+            (a.gshape[-2], b.gshape[1]), split=split, dtype=c_type, device=a.device, comm=a.comm
+        )
         c.larray += a.larray @ b.larray
 
         ret = c if not vector_flag else c.squeeze()
@@ -582,7 +585,9 @@ def matmul(a: DNDarray, b: DNDarray, allow_resplit: bool = False) -> DNDarray:
         c += a.larray @ b.larray[a_idx[1].start : a_idx[1].start + a.lshape[-1], :]
         a.comm.Allreduce(MPI.IN_PLACE, c, MPI.SUM)
         c = c if not vector_flag else c.squeeze()
-        ret = factories.array(c, split=a.split if b.gshape[1] > 1 else 0, device=a.device)
+        ret = factories.array(
+            c, split=a.split if b.gshape[1] > 1 else 0, device=a.device, comm=a.comm
+        )
         if gpu_int_flag:
             ret = og_type(ret, device=a.device)
         return ret
@@ -593,7 +598,9 @@ def matmul(a: DNDarray, b: DNDarray, allow_resplit: bool = False) -> DNDarray:
         c += a.larray[:, b_idx[0].start : b_idx[0].start + b.lshape[0]] @ b.larray
         b.comm.Allreduce(MPI.IN_PLACE, c, MPI.SUM)
         c = c if not vector_flag else c.squeeze()
-        ret = factories.array(c, split=b.split if a.gshape[-2] > 1 else 0, device=a.device)
+        ret = factories.array(
+            c, split=b.split if a.gshape[-2] > 1 else 0, device=a.device, comm=a.comm
+        )
         if gpu_int_flag:
             ret = og_type(ret, device=a.device)
         return ret
@@ -608,7 +615,7 @@ def matmul(a: DNDarray, b: DNDarray, allow_resplit: bool = False) -> DNDarray:
         c = c if not vector_flag else c.squeeze()
         split = a.split if b.gshape[1] > 1 else 0
         split = split if not vector_flag else 0
-        ret = factories.array(c, split=split, device=a.device)
+        ret = factories.array(c, split=split, device=a.device, comm=a.comm)
         if gpu_int_flag:
             ret = og_type(ret, device=a.device)
         return ret
@@ -619,7 +626,7 @@ def matmul(a: DNDarray, b: DNDarray, allow_resplit: bool = False) -> DNDarray:
         c = c if not vector_flag else c.squeeze()
         split = b.split if a.gshape[1] > 1 else 0
         split = split if not vector_flag else 0
-        ret = factories.array(c, is_split=split, device=a.device)
+        ret = factories.array(c, is_split=split, device=a.device, comm=a.comm)
         if gpu_int_flag:
             ret = og_type(ret, device=a.device)
         return ret
@@ -695,10 +702,10 @@ def matmul(a: DNDarray, b: DNDarray, allow_resplit: bool = False) -> DNDarray:
 
     # for the communication scheme, the output array needs to be created
     c_shape = (a.gshape[-2], b.gshape[1])
-    c = factories.zeros(c_shape, split=a.split, dtype=c_type, device=a.device)
+    c = factories.zeros(c_shape, split=a.split, dtype=c_type, device=a.device, comm=a.comm)
 
     # get the index map for c
-    c_index_map = factories.zeros((c.comm.size, 2, 2), device=a.device)
+    c_index_map = factories.zeros((c.comm.size, 2, 2), device=a.device, comm=a.comm)
     c_idx = c.comm.chunk(c.shape, c.split)[2]
     c_index_map[c.comm.rank, 0, :] = (c_idx[0].start, c_idx[0].stop)
     c_index_map[c.comm.rank, 1, :] = (c_idx[1].start, c_idx[1].stop)
@@ -919,7 +926,7 @@ def matmul(a: DNDarray, b: DNDarray, allow_resplit: bool = False) -> DNDarray:
             if c_loc.nelement() == 1:
                 c_loc = torch.tensor(c_loc, device=tdev)
 
-            c = factories.array(c_loc, is_split=0, device=a.device)
+            c = factories.array(c_loc, is_split=0, device=a.device, comm=a.comm)
         if gpu_int_flag:
             c = og_type(c, device=a.device)
         return c
@@ -1023,7 +1030,7 @@ def matmul(a: DNDarray, b: DNDarray, allow_resplit: bool = False) -> DNDarray:
                     c.larray[:, : b_node_rem_s1.shape[1]] += a_rem @ b_node_rem_s1
                 del a_lp_data[pr]
         if vector_flag:
-            c = factories.array(c.larray.squeeze(), is_split=0, device=a.device)
+            c = factories.array(c.larray.squeeze(), is_split=0, device=a.device, comm=a.comm)
         if gpu_int_flag:
             c = og_type(c, device=a.device)
         return c
@@ -1066,7 +1073,7 @@ def matmul(a: DNDarray, b: DNDarray, allow_resplit: bool = False) -> DNDarray:
                 c.larray[: sp0 - st0, st1:sp1] += a.larray @ b_lp_data[pr]
                 del b_lp_data[pr]
         if vector_flag:
-            c = factories.array(c.larray.squeeze(), is_split=0, device=a.device)
+            c = factories.array(c.larray.squeeze(), is_split=0, device=a.device, comm=a.comm)
         if gpu_int_flag:
             c = og_type(c, device=a.device)
 
@@ -1090,7 +1097,7 @@ def matmul(a: DNDarray, b: DNDarray, allow_resplit: bool = False) -> DNDarray:
         if vector_flag:
             split = 0
             res = res.squeeze()
-        c = factories.array(res, split=split, device=a.device)
+        c = factories.array(res, split=split, device=a.device, comm=a.comm)
         if gpu_int_flag:
             c = og_type(c, device=a.device)
         return c