Add workflow scripts for github actions

.github/workflows/build-one-job.yml

@@ -0,0 +1,154 @@
+name: Dragon
+on:
+  workflow_dispatch: {}
+  # push:
+  #   branches:
+  #     - master
+  #pull_request:
+  #  types:
+  #    - opened
+  #    - synchronize
+  #    - reopened
+  #  branches:
+  #    - main
+
+jobs:
+  Build:
+
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: 
+        false
+      matrix:
+        python-version: ["3.9.4", "3.10.10", "3.11.7"]
+    outputs:
+        GITHASH: ${{ steps.build.outputs.GITHASH }}
+        VERSION: ${{ steps.build.outputs.VERSION }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: check environment
+        run: |
+          set -x
+          uname -a 
+          git branch 
+          ls 
+          which python3 
+          python3 --version
+          whoami
+
+      - name: Build 
+        id: build
+        run: |
+          . hack/build_no_module
+          set -x
+          ARTIFACT=$(ls ./src/release)
+          GITHASH=$(git rev-parse --short HEAD)
+          VERSION=$(echo $ARTIFACT | awk -F- '{print $2}')
+          echo "GITHASH=$GITHASH" >> $GITHUB_OUTPUT
+          echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
+          mv ./src/release/$ARTIFACT ./src/release/dragon-$VERSION-py${{ matrix.python-version }}-$GITHASH.tar.gz
+          ARTIFACT=$(ls ./src/release)
+          echo "ARTIFACT=$ARTIFACT" >> $GITHUB_OUTPUT
+
+      - name: Run Test
+        run: |
+          set -ex
+          pwd 
+          ls -la
+          ls ./src/release
+          echo "ARTIFACT=$ARTIFACT"
+
+          ############################
+          ### untar relase package ###
+          ############################
+          mkdir -p release
+          tar -C release -xvzf $PWD/src/release/${{ steps.build.outputs.ARTIFACT }}        
+          ROOTDIR="$(realpath release/dragon-${{ steps.build.outputs.VERSION }})"
+
+          #################################
+          ### Install Python Dependency ###
+          #################################
+          python3 -m venv dvenv 
+          source dvenv/bin/activate
+          python3 -m pip install -U pip
+          python3 -m pip install -c src/constraints.txt \
+              cloudpickle \
+              cryptography \
+              numpy \
+              parameterized \
+              scipy \
+              wheel \
+              pyyaml
+          python3 -m pip install $ROOTDIR/dragon-${{ steps.build.outputs.VERSION }}-*.whl
+
+
+          ############################
+          ### set environment path ### 
+          ############################
+          pushd $ROOTDIR
+          export DRAGON_VERSION=${{ steps.build.outputs.VERSION }}
+          export DRAGON_BASE_DIR=$PWD
+          export DRAGON_INCLUDE_DIR=$DRAGON_BASE_DIR/include
+          export DRAGON_LIB_DIR=$DRAGON_BASE_DIR/lib
+          export PATH=$DRAGON_BASE_DIR/bin:$PATH
+          export LD_LIBRARY_PATH=$DRAGON_LIB_DIR:$LD_LIBRARY_PATH
+          export LIBRARY_PATH=$DRAGON_LIB_DIR:$LIBRARY_PATH
+          export DRAGON_DEBUG="0"
+          popd
+
+
+          ####################
+          ### Run unittest ###
+          ####################
+          make -C test test
+
+          find /dev/shm  -user $(USER) -exec rm -fr {} \;
+          pushd $ROOTDIR/dragon_unittests
+          python3 test_utils.py -f -v
+          python3 test_channels.py -f -v
+          popd
+
+          # Run Dragon GS client API tests
+          find /dev/shm  -user $(USER) -exec rm -fr {} \;
+          pushd $ROOTDIR/examples/dragon_gs_client/
+          dragon queue_demo.py
+          dragon pi_demo.py 4
+          dragon --single-node-override pi_demo.py 4
+          # Reduce size of default memory pool to 0.5GB
+          DRAGON_DEFAULT_SEG_SZ=536870912 dragon connection_demo.py
+          dragon dragon_run_api.py ls
+          dragon dragon_popen_api.py ls
+          popd
+
+          find /dev/shm  -user $(USER) -exec rm -fr {} \;
+          pushd $ROOTDIR/examples/dragon_native/
+          dragon pi_demo.py 4
+          popd
+
+          find /dev/shm  -user $(USER) -exec rm -fr {} \;
+          pushd $ROOTDIR/examples/multiprocessing/unittests
+          make
+          popd
+
+          # Run multiprocessing benchmarks
+          find /dev/shm  -user $(USER) -exec rm -fr {} \;
+          pushd $ROOTDIR/examples/multiprocessing/
+          dragon p2p_lat.py --iterations 1000 --lg_max_message_size 12 --dragon
+          dragon p2p_bw.py --iterations 10 --lg_max_message_size 12 --dragon
+          dragon aa_bench.py --iterations 100 --num_workers $NCPU --lg_message_size 21 --dragon
+          popd
+
+          # Run multiprocessing numpy teests
+          find /dev/shm  -user $(USER) -exec rm -fr {} \;
+          pushd $ROOTDIR/examples/multiprocessing/numpy-mpi4py-examples
+          dragon numpy_scale_work.py
+          dragon numpy_scale_work.py --dragon
+          dragon scipy_scale_work.py
+          dragon scipy_scale_work.py --dragon
+          popd
+

.github/workflows/build.yml

@@ -0,0 +1,192 @@
+name: Build and Test
+on:
+  workflow_dispatch: {}
+  # push:
+  #   branches:
+  #     - build-workflow
+#   push:
+#     branches:
+#       - main
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+    branches:
+      - main
+
+jobs:
+  Build:
+
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: 
+        false
+      matrix:
+        python-version: ["3.11"]
+        # python-version: ["3.9", "3.10", "3.11"]
+    outputs:
+        GITHASH: ${{ steps.build.outputs.GITHASH }}
+        VERSION: ${{ steps.build.outputs.VERSION }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: check environment
+        run: |
+          set -x
+          uname -a 
+          git branch 
+          ls 
+          which python3 
+          python3 --version
+          whoami
+          sudo apt-get install libnuma-dev
+      - name: Build 
+        id: build
+        run: |
+          . hack/build_no_module
+          set -x
+          ARTIFACT=$(ls ./src/release)
+          GITHASH=$(git rev-parse --short HEAD)
+          VERSION=$(echo $ARTIFACT | awk -F- '{print $2}')
+          echo "GITHASH=$GITHASH" >> $GITHUB_OUTPUT
+          echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
+          mv ./src/release/$ARTIFACT ./src/release/dragon-$VERSION-py${{ matrix.python-version }}-$GITHASH.tar.gz
+          ARTIFACT=$(ls ./src/release)
+          echo "ARTIFACT=$ARTIFACT" >> $GITHUB_OUTPUT
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v4
+        with:
+            name: ${{ steps.build.outputs.ARTIFACT }}
+            path: src/release/${{ steps.build.outputs.ARTIFACT }}
+
+  Test:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: 
+        false
+      matrix:
+        python-version: ["3.11"]
+        # python-version: ["3.9", "3.10", "3.11"]
+    needs: Build
+    env:
+        GITHASH: ${{ needs.Build.outputs.GITHASH }}
+        VERSION: ${{ needs.Build.outputs.VERSION }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}         
+      - name: Download Artifacts
+        uses: actions/download-artifact@v4
+        with:
+            name: dragon-${{ env.VERSION }}-py${{ matrix.python-version }}-${{ env.GITHASH }}.tar.gz
+      - name: Run Test
+        run: |
+          set -x
+          pwd
+          ls -la
+          # run unittest
+
+          ############################
+          ### untar relase package ###
+          ############################
+          ARTIFACT=dragon-${{ env.VERSION }}-py${{ matrix.python-version }}-${{ env.GITHASH }}.tar.gz
+          mkdir -p release
+          tar -C release -xvzf $ARTIFACT
+          ROOTDIR="$(realpath release/dragon-${{ env.VERSION }})"
+
+          #################################
+          ### Install Python Dependency ###
+          #################################
+          python3 -m venv dvenv 
+          source dvenv/bin/activate
+          python3 -m pip install -U pip
+          python3 -m pip install -c src/constraints.txt \
+              cloudpickle \
+              cryptography \
+              numpy \
+              parameterized \
+              scipy \
+              wheel \
+              pyyaml
+          python3 -m pip install $ROOTDIR/dragon-${{ env.VERSION }}-*.whl
+          python3 -m pip install $ROOTDIR/pycapnp*.whl
+
+
+          ############################
+          ### set environment path ### 
+          ############################
+          pushd $ROOTDIR
+          export DRAGON_VERSION=${{ steps.build.outputs.VERSION }}
+          export DRAGON_BASE_DIR=$PWD
+          export DRAGON_INCLUDE_DIR=$DRAGON_BASE_DIR/include
+          export DRAGON_LIB_DIR=$DRAGON_BASE_DIR/lib
+          export PATH=$DRAGON_BASE_DIR/bin:$PATH
+          export LD_LIBRARY_PATH=$DRAGON_LIB_DIR:$LD_LIBRARY_PATH
+          export LIBRARY_PATH=$DRAGON_LIB_DIR:$LIBRARY_PATH
+          export DRAGON_DEBUG="0"
+          popd
+
+
+          ####################
+          ### Run unittest ###
+          ####################
+          echo "---------------------Running C Tests"
+          make -C test test
+
+          find /dev/shm  -exec rm -fr {} \;
+          pushd $ROOTDIR/dragon_unittests
+          #echo "---------------------Running basic unittests"
+          python3 test_utils.py -f -v
+          python3 test_channels.py -f -v
+          echo "---------------------Basic unittests done"
+          popd
+
+          # Run Dragon GS client API tests
+          find /dev/shm  -exec rm -fr {} \;
+          pushd $ROOTDIR/examples/dragon_gs_client/
+          dragon queue_demo.py
+          dragon pi_demo.py 4
+          dragon --single-node-override pi_demo.py 4
+          # Reduce size of default memory pool to 0.5GB
+          DRAGON_DEFAULT_SEG_SZ=536870912 dragon connection_demo.py
+          dragon dragon_run_api.py ls
+          dragon dragon_popen_api.py ls
+          popd
+
+          find /dev/shm -exec rm -fr {} \;
+          pushd $ROOTDIR/examples/dragon_native/
+          dragon pi_demo.py 4
+          popd
+
+          find /dev/shm  -exec rm -fr {} \;
+          pushd $ROOTDIR/examples/multiprocessing/unittests
+          echo "---------------------Running multiprocessing unittests"
+          make
+          popd
+
+          # Run multiprocessing benchmarks
+          find /dev/shm -exec rm -fr {} \;
+          pushd $ROOTDIR/examples/multiprocessing/
+          echo "---------------------Running multiprocessing benchmark examples"
+          dragon p2p_lat.py --iterations 1000 --lg_max_message_size 12 --dragon
+          dragon p2p_bw.py --iterations 10 --lg_max_message_size 12 --dragon
+          dragon aa_bench.py --iterations 100 --num_workers 4 --lg_message_size 21 --dragon
+          popd
+
+          # Run multiprocessing numpy teests
+          find /dev/shm -exec rm -fr {} \;
+          pushd $ROOTDIR/examples/multiprocessing/numpy-mpi4py-examples
+          echo "---------------------Running multiprocessing numpy tests"
+          dragon numpy_scale_work.py
+          dragon numpy_scale_work.py --dragon
+          dragon scipy_scale_work.py
+          dragon scipy_scale_work.py --dragon
+          popd
+
+

hack/build_no_module

@@ -0,0 +1,45 @@
+
+export DRAGON_VERSION=0.61
+export DRAGON_BASE_DIR=$PWD/src
+echo "DRAGON_BASE_DIR=$DRAGON_BASE_DIR"
+export DRAGON_INCLUDE_DIR=$DRAGON_BASE_DIR/include
+export DRAGON_LIB_DIR=$DRAGON_BASE_DIR/lib
+export DRAGON_BUILD_NTHREADS=6
+export PYTHONPATH=$DRAGON_BASE_DIR
+##########################################################
+### alias should already been take care of in src/setup.py ###
+########################################################## 
+# alias dragon="python3 -m dragon.cli dragon"
+# alias dragon-backend="python3 -m dragon.cli dragon-backend"
+# alias dragon-localservices="python3 -m dragon.cli dragon-localservices"
+# alias dragon-globalservices="python3 -m dragon.cli dragon-globalservices"
+# alias dragon-tcp="python3 -m dragon.cli dragon-tcp"
+# alias dragon-hsta-if="python3 -m dragon.cli dragon-hsta-if"
+
+
+pythonpath=`which python3`
+echo $pythonpath
+env_str='_env'
+echo $env_str
+if [[ "$pythonpath" == *"$env_str"* ]]; then
+  echo "Deactivating environment."
+  deactivate
+fi
+
+echo "Building and activating new, clean environment."
+python3 -m venv --clear _env
+. _env/bin/activate
+python3 -m pip install -U pip
+python3 -m pip install -r src/requirements.txt -c src/constraints.txt
+export PATH=$PWD/hack:$PATH
+
+echo "Building externals"
+cd external
+make clean
+make build-capnproto
+cd ..
+
+echo "Building source code."
+cd src
+make dist
+cd ..