Merge pull request #3016 from radical-cybertools/devel_nodb_2

Devel nodb 2
radical-cybertools · Sep 27, 2023 · 1b081a0 · 1b081a0
2 parents c8a2705 + 5a517f1
commit 1b081a0
Show file tree

Hide file tree

Showing 159 changed files with 5,659 additions and 4,966 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -3,11 +3,14 @@ name: CI
 
 on:
   push:
-    branches: [ devel ]
+    branches:
+      - 'devel*'
   pull_request:
-    branches: [ devel ]
+    branches:
+      - 'devel*'
 
 jobs:
+
   linting:
     runs-on: ubuntu-latest
     steps:
@@ -22,10 +25,7 @@ jobs:
           python -m venv testenv
           . testenv/bin/activate
           python -m pip install --upgrade pip setuptools wheel
-          python -m pip install git+https://github.com/radical-cybertools/radical.utils.git@devel
-          python -m pip install git+https://github.com/radical-cybertools/radical.saga.git@devel
-          python -m pip install --upgrade .
-          python -m pip install -r requirements-tests.txt
+          python -m pip install -r requirements-ci.txt
       - name: Lint with flake8 and pylint
         run: |
           . testenv/bin/activate
@@ -41,22 +41,8 @@ jobs:
           flake8 $FILTERED
           pylint $FILTERED
 
-  notebook_integration_test:
-    strategy:
-      matrix:
-        python-version: [ '3.7', '3.8', '3.9', '3.10', '3.11' ]
-    uses: ./.github/workflows/run-rp-notebook.yml
-    with:
-      python-version: ${{ matrix.python-version }}
-      notebook-name: 'getting_started.ipynb'
-
   tests:
     runs-on: ${{ matrix.os }}
-    services:
-      mongodb:
-        image: mongo
-        ports:
-        - 27017/tcp
     strategy:
       matrix:
         os: [ ubuntu-latest ]
@@ -65,53 +51,40 @@ jobs:
           - os: ubuntu-20.04
             python-version: '3.6'
     steps:
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 2
-    - uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        python -m venv testenv
-        . testenv/bin/activate
-        python -m pip install --upgrade pip setuptools wheel
-        python -m pip install git+https://github.com/radical-cybertools/radical.utils.git@devel
-        python -m pip install git+https://github.com/radical-cybertools/radical.saga.git@devel
-        python -m pip install --upgrade .
-        python -m pip install -r requirements-tests.txt
-    - name: Unit tests
-      env:
-        RADICAL_PILOT_DBURL:  mongodb://localhost:${{ job.services.mongodb.ports[27017] }}/test
-      run: |
-        . testenv/bin/activate
-        coverage run --source=radical.pilot -m pytest -ra -vvv --showlocals tests/unit_tests/ tests/component_tests/
-        coverage xml
-    - uses: codecov/codecov-action@v3
-      if: ${{ matrix.python-version == '3.7' }}
-      with:
-        files: ./coverage.xml
-    # - name: Integration test
-    #   env:
-    #     MONGODB_HOST: localhost
-    #     MONGODB_PORT: ${{ job.services.mongodb.ports[27017] }}
-    #     RADICAL_PILOT_DBURL:  mongodb://localhost:${{ job.services.mongodb.ports[27017] }}/test
-    #     TARGET_PATH: 'docs/source/getting_started.ipynb'
-    #   run: |
-    #     . testenv/bin/activate
-    #     python -m pip install -r requirements-docs.txt
-    #     python -m pip install jupyter
-    #     jupyter nbconvert --clear-output --inplace $TARGET_PATH
-    #     jupyter nbconvert --to notebook --execute --inplace $TARGET_PATH
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m venv testenv
+          . testenv/bin/activate
+          python -m pip install --upgrade pip setuptools wheel
+          python -m pip install -r requirements-ci.txt
+      - name: Unit tests
+        run: |
+          . testenv/bin/activate
+          coverage run --source=radical.pilot -m pytest -ra -vvv --showlocals tests/unit_tests/ tests/component_tests/
+          coverage xml
+      - uses: codecov/codecov-action@v3
+        if: ${{ matrix.python-version == '3.7' }}
+        with:
+          files: ./coverage.xml
 
+  notebook_integration_test:
+    strategy:
+      matrix:
+        python-version: [ '3.7', '3.8', '3.9', '3.10', '3.11' ]
+    uses: ./.github/workflows/run-rp-notebook.yml
+    with:
+      python-version: ${{ matrix.python-version }}
+      requirements-file: 'requirements-docs-ci.txt'
+      notebook-name: 'getting_started.ipynb'
 
   analytics:
     runs-on: ${{ matrix.os }}
-    services:
-      mongodb:
-        image: mongo
-        ports:
-        - 27017/tcp
     strategy:
       matrix:
         os: [ ubuntu-latest ]
@@ -120,41 +93,36 @@ jobs:
           - os: ubuntu-20.04
             python-version: '3.6'
     steps:
-    - uses: actions/checkout@v3
-      with:
-        fetch-depth: 2
-    - uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        sudo apt update
-        sudo apt install -y texlive cm-super
-        sudo apt install -y texlive-fonts-extra texlive-extra-utils dvipng
-        sudo apt install -y texlive-fonts-recommended texlive-latex-extra
-        python -m venv testenv
-        . testenv/bin/activate
-        python -m pip install --upgrade pip setuptools wheel
-        python -m pip install git+https://github.com/radical-cybertools/radical.utils.git@devel
-        python -m pip install git+https://github.com/radical-cybertools/radical.saga.git@devel
-        python -m pip install --upgrade .
-        python -m pip install git+https://github.com/radical-cybertools/radical.analytics.git@devel
-    - name: analyze example session
-      timeout-minutes: 5
-      env:
-        RADICAL_PILOT_DBURL:  mongodb://localhost:${{ job.services.mongodb.ports[27017] }}/test
-      run: |
-        . testenv/bin/activate
-        ./examples/00_getting_started.py local.localhost
-        SID=$(ls -rt | grep rp.session)
-        echo "$SID: $SID"
-        radical-analytics-inspect "$SID"
-        mkdir artifacts
-        ls -la
-        cp -R *.png *.stats artifacts
-    - name: upload artifacts
-      uses: actions/upload-artifact@v3
-      with:
-          name: artifacts
-          path: artifacts
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y texlive cm-super
+          sudo apt install -y texlive-fonts-extra texlive-extra-utils dvipng
+          sudo apt install -y texlive-fonts-recommended texlive-latex-extra
+          python -m venv testenv
+          . testenv/bin/activate
+          python -m pip install --upgrade pip setuptools wheel
+          python -m pip install -r requirements-ci.txt
+      - name: analyze example session
+        timeout-minutes: 5
+        run: |
+          . testenv/bin/activate
+          ./examples/00_getting_started.py local.localhost
+          SID=$(ls -rt | grep rp.session)
+          echo "$SID: $SID"
+          radical-analytics-inspect "$SID"
+          mkdir artifacts
+          ls -la
+          cp -R *.png *.stats artifacts
+      - name: upload artifacts
+        uses: actions/upload-artifact@v3
+        with:
+            name: artifacts
+            path: artifacts
 
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -9,10 +9,10 @@ name: 'Test Jupyter notebooks'
 on:
   push:
     branches:
-      - docs/nb_section3
+      - 'devel*'
   pull_request:
     branches:
-      - docs/nb_section3
+      - 'devel*'
 
 # This allows a subsequently queued workflow run to interrupt previous runs
 concurrency:
@@ -24,23 +24,26 @@ jobs:
     uses: ./.github/workflows/run-rp-notebook.yml
     with:
       python-version: 3.7
+      requirements-file: 'requirements-docs-ci.txt'
       notebook-name: 'getting_started.ipynb'
 
   test-tutorials:
     strategy:
       matrix:
         tutorial: [
           'configuration.ipynb',
-          # 'debugging.ipynb',
+          'debugging.ipynb',
           'describing_tasks.ipynb',
           'multiple_pilots.ipynb',
-          # 'profiling.ipynb',
-          # 'raptor.ipynb',
+          'profiling.ipynb',
+          'raptor.ipynb',
           'staging_data.ipynb',
           'submission.ipynb'
         ]
     uses: ./.github/workflows/run-rp-notebook.yml
     with:
       python-version: 3.7
+      requirements-file: 'requirements-docs-ci.txt'
       notebook-name: ${{ matrix.tutorial }}
-      notebook-path: 'tutorials'
+      notebook-path: 'tutorials'
+
diff --git a/.github/workflows/run-rp-notebook.yml b/.github/workflows/run-rp-notebook.yml
@@ -7,6 +7,11 @@ on:
         description: 'Python version for running the Jupyter notebook'
         required: true
         type: string
+      requirements-file:
+        description: 'File with dependencies'
+        required: false
+        default: 'requirements-docs.txt'
+        type: string
       notebook-name:
         description: 'File name of the Jupyter notebook'
         required: true
@@ -25,11 +30,6 @@ on:
 jobs:
   tests:
     runs-on: ubuntu-latest
-    services:
-      mongodb:
-        image: mongo
-        ports:
-          - 27017/tcp    # will assign a random free host port
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
@@ -41,18 +41,39 @@ jobs:
           python-version: ${{ inputs.python-version }}
       - name: Install dependencies
         run: |
+          sudo apt update -y && sudo apt install -y mpich
           python -m venv testenv
           . testenv/bin/activate
-          python -m pip install --upgrade pip
-          python -m pip install -r requirements-docs.txt
+          python -m pip install --upgrade pip setuptools wheel
+          python -m pip install -r ${{ inputs.requirements-file }}
           python -m pip install jupyter
       - name: Run Jupyter Notebook
         env:
-          MONGODB_HOST: localhost
-          MONGODB_PORT: ${{ job.services.mongodb.ports[27017] }}
-          RADICAL_PILOT_DBURL:  mongodb://localhost:${{ job.services.mongodb.ports[27017] }}/test
           TARGET_PATH: ${{ format('{0}/{1}/{2}', inputs.documentation-path, inputs.notebook-path, inputs.notebook-name) }}
+        timeout-minutes: 5
+        # continue-on-error: true
         run: |
           . testenv/bin/activate
           jupyter nbconvert --clear-output --inplace $TARGET_PATH
           jupyter nbconvert --to notebook --execute --inplace $TARGET_PATH
+      - name: Collect session
+        if: always()
+        run: |
+          SIDCLIENT=$(ls -rt | grep rp.session)
+          SIDAGENT="$HOME/radical.pilot.sandbox/$SIDCLIENT"
+          CLIENTNAME="${{inputs.notebook-name}}_client_$SIDCLIENT"
+          AGENTNAME="${{inputs.notebook-name}}_agent_$SIDCLIENT"
+          mkdir session
+          tar cvfj $CLIENTNAME.tar.bz2 $SIDCLIENT
+          cp -R $CLIENTNAME.tar.bz2 session
+          if [ -d "$SIDAGENT" ]; then
+            tar cvfj $AGENTNAME.tar.bz2 $SIDAGENT
+            cp -R $AGENTNAME.tar.bz2 session
+          fi
+      - name: upload session
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+            name: session
+            path: session
+            retention-days: 5
diff --git a/.readthedocs.yml b/.readthedocs.yml
@@ -3,9 +3,14 @@ version: 2
 
 formats: [htmlzip]
 
+build:
+  os: "ubuntu-22.04"
+  tools:
+    python: "3.7"
+  apt_packages:
+    - mpich
+
 python:
-  system_packages: true
-  version: 3.7
   install:
     - requirements: requirements-docs.txt
     - method: pip

diff --git a/TODO b/TODO
@@ -252,9 +252,14 @@ term iv
   - Exception in SA startup causes hang (agent_0 does not die)
 
 
+others
+------
   - `PRTE` switches in scheduler should become `partition` switches, where the
     partitions are defined by the RM
 
+  - stager and other RPC like workers should get a proper async RPC channel
+    (req/res).  That should be built upon `ru.zmq.Service`.  The client side
+    should wrap the request into a proper async Future.
 
   - create_named_env needs to issue a client error on failures
 

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-1.38.0
+1.40.0