Prepare sdk release 3.3 (#36)

All changes for 3.3 SDK

Author: rahult-graphcore

.github/deployment_configs/deploy-pyg-paperspace.yaml

@@ -0,0 +1,193 @@
+# Copyright (c) 2022 Graphcore Ltd. All rights reserved.
+
+_common_source_repository: &_common_source_repository
+  origin: git@github.com:graphcore/examples-internal.git
+  ref: sdk-release-3.3.0
+
+_common_target_repository: &_common_target_repository
+  origin: notebooks/
+  ref: null
+
+
+pyg-cluster-GCN:
+  source:
+    paths:
+    - path: gnn/cluster_gcn/pytorch_geometric/node_classification_with_cluster_gcn.ipynb
+    - path: gnn/cluster_gcn/pytorch_geometric/requirements.txt
+    repository:
+      <<: *_common_source_repository
+      prefix: gnn/cluster_gcn/pytorch_geometric
+  target:
+    renames: {node-prediction/node_classification_with_cluster_gcn.ipynb: node-prediction/cluster_gcn_training.ipynb}
+    repository:
+      <<: *_common_target_repository
+      prefix: node-prediction
+
+
+pyg-message-passing:
+  source:
+    paths:
+    - path: gnn/message_passing/pytorch_geometric/molecular_property_prediction_with_gin.ipynb
+    - path: gnn/message_passing/pytorch_geometric/model.py
+    - path: gnn/message_passing/pytorch_geometric/requirements.txt
+    repository:
+      <<: *_common_source_repository
+      prefix: gnn/message_passing/pytorch_geometric
+  target:
+    renames: {graph-prediction/gin-molecular-prediction/molecular_property_prediction_with_gin.ipynb: graph-prediction/gin-molecular-prediction/gin_training.ipynb}
+    repository:
+      <<: *_common_target_repository
+      prefix: graph-prediction/gin-molecular-prediction/
+
+pyg-schnet:
+  source:
+    paths:
+    - path: gnn/schnet/pytorch_geometric/molecular_property_prediction_with_schnet.ipynb
+    - path: gnn/schnet/pytorch_geometric/utils.py
+    - path: gnn/schnet/pytorch_geometric/requirements.txt
+    repository:
+      <<: *_common_source_repository
+      prefix: gnn/schnet/pytorch_geometric/
+  target:
+    renames: {graph-prediction/schnet-molecular-prediction/molecular_property_prediction_with_schnet.ipynb: graph-prediction/schnet-molecular-prediction/schnet_training.ipynb }
+    repository:
+      <<: *_common_target_repository
+      prefix: graph-prediction/schnet-molecular-prediction/
+
+pyg-tgn:
+  source:
+    paths:
+    - expression: '*'
+      path: gnn/tgn/pytorch
+      recursive: true
+    excludes:
+    - path: gnn/tgn/pytorch/README.md
+    - path: gnn/tgn/pytorch/.gitignore
+    - expression: '*'
+      path: gnn/tgn/pytorch/.ci
+      recursive: true
+    repository:
+      <<: *_common_source_repository
+      prefix: gnn/tgn/pytorch
+  target:
+    renames: {}
+    repository:
+      <<: *_common_target_repository
+      prefix: link-prediction/temporal-graph-networks
+
+pyg-nbfnet-notebook:
+  source:
+    paths:
+    - path: gnn/nbfnet/pytorch_geometric/NBFNet_training.ipynb
+    - path: gnn/nbfnet/pytorch_geometric/requirements.txt
+    repository:
+      <<: *_common_source_repository
+      prefix: gnn/nbfnet/pytorch_geometric
+  target:
+    renames: {}
+    repository:
+      <<: *_common_target_repository
+      prefix: link-prediction/NBFNet-for-knowledge-graphs/
+
+pyg-nbfnet-code:
+  source:
+    paths:
+    - expression: '*'
+      path: gnn/nbfnet/pytorch_geometric
+      recursive: true
+    excludes:
+    - path: gnn/nbfnet/pytorch_geometric/NBFNet_training.ipynb
+    - expression: '*'
+      path: gnn/nbfnet/pytorch_geometric/.ci
+      recursive: true
+    repository:
+      <<: *_common_source_repository
+      prefix: gnn/nbfnet/pytorch_geometric
+  target:
+    renames: {}
+    repository:
+      <<: *_common_target_repository
+      prefix: link-prediction/NBFNet-for-knowledge-graphs/nbfnet_utils/
+
+# BENCHMARK
+pyg-benchmark:
+  source:
+    paths:
+    - expression: '*'
+      path: benchmarks/
+      recursive: true
+    repository:
+      origin: https://github.com/graphcore/poppyg.git
+      ref: adams/benchmark_gather_scatter
+      prefix: benchmarks/
+  target:
+    renames: {benchmarks/benchmarking_message_passing_on_the_ipu.ipynb: benchmarks/benchmarking_gnn_on_ipus.ipynb}
+    repository:
+      <<: *_common_target_repository
+      prefix: ipu-speed-advantage/
+
+# tutorials
+pyg-1_at_a_glance:
+  source:
+    paths:
+    - path: tutorials/tutorials/pytorch_geometric/1_at_a_glance/1_at_a_glance.ipynb
+    repository:
+      <<: *_common_source_repository
+      prefix: tutorials/tutorials/pytorch_geometric/1_at_a_glance/
+  target:
+    renames: {}
+    repository:
+      <<: *_common_target_repository
+      prefix: learning-pytorch-geometric-on-ipus/
+
+pyg-2_a_worked_example:
+  source:
+    paths:
+    - path: tutorials/tutorials/pytorch_geometric/2_a_worked_example/2_a_worked_example.ipynb
+    repository:
+      <<: *_common_source_repository
+      prefix: tutorials/tutorials/pytorch_geometric/2_a_worked_example/
+  target:
+    renames: {}
+    repository:
+      <<: *_common_target_repository
+      prefix: learning-pytorch-geometric-on-ipus/
+
+pyg-3_small_graph_batching_with_padding:
+  source:
+    paths:
+    - path: tutorials/tutorials/pytorch_geometric/3_small_graph_batching_with_padding/3_small_graph_batching_with_padding.ipynb
+    repository:
+      <<: *_common_source_repository
+      prefix: tutorials/tutorials/pytorch_geometric/3_small_graph_batching_with_padding/
+  target:
+    renames: {}
+    repository:
+      <<: *_common_target_repository
+      prefix: learning-pytorch-geometric-on-ipus/
+
+pyg-4_small_graph_batching_with_packing:
+  source:
+    paths:
+    - path: tutorials/tutorials/pytorch_geometric/4_small_graph_batching_with_packing/4_small_graph_batching_with_packing.ipynb
+    repository:
+      <<: *_common_source_repository
+      prefix: tutorials/tutorials/pytorch_geometric/4_small_graph_batching_with_packing/
+  target:
+    renames: {}
+    repository:
+      <<: *_common_target_repository
+      prefix: learning-pytorch-geometric-on-ipus/
+
+pyg-tutorial-requirements:
+  source:
+    paths:
+    - path: tutorials/tutorials/pytorch_geometric/requirements.txt
+    repository:
+      <<: *_common_source_repository
+      prefix: tutorials/tutorials/pytorch_geometric/
+  target:
+    renames: {}
+    repository:
+      <<: *_common_target_repository
+      prefix: learning-pytorch-geometric-on-ipus/

.github/workflows/2-local-vpod-tests.yml

@@ -15,7 +15,7 @@ on:
         type: string
         description: "Docker image used in notebook testing"
         required: false
-        default: "graphcore/pytorch-geometric-jupyter:3.2.1-ubuntu-20.04-20230531"
+        default: "localhost:5000/graphcore/pytorch-geometric-paperspace:3.3.0"
       notebooks:
         type: string
         description: "List of notebooks to test in JSON format"
@@ -45,7 +45,7 @@ jobs:
   tests:
     uses: graphcore/paperspace-automation/.github/workflows/subwf-vpod-tests-for-nb-repo.yml@main
     with:
-      docker_image: ${{ inputs.docker_image || 'graphcore/pytorch-geometric-jupyter:3.2.1-ubuntu-20.04-20230531' }}
+      docker_image: ${{ inputs.docker_image || 'localhost:5000/graphcore/pytorch-geometric-paperspace:3.3.0' }}
       notebooks: ${{ inputs.notebooks }}
       machine_types: ${{ inputs.machine_types }}
       test_env: ${{ inputs.test_env || 'Local' }}

.github/workflows/copy-notebooks-from-source.yml

@@ -0,0 +1,20 @@
+name: Copy notebooks from source repos
+
+on:
+  workflow_dispatch:
+    inputs:
+      deployment_spec:
+        required: false
+        type: string
+        description: "Spec file to be read for copying notebooks, must be valid input to deployment/deploy.py"
+        default: .github/deployment-configs/deploy-pyg-paperspace.yaml
+  pull_request:
+
+
+jobs:
+  tests:
+    uses: graphcore/paperspace-automation/.github/workflows/copy-to-nb-repo.yml@main
+    with:
+      deployment_spec: ${{ inputs.deployment_spec }}
+    secrets:
+      gh_token:  ${{ secrets.GH_TOKEN_SYNC_REPOS }}

.gitignore

@@ -1 +1 @@
-.ipynb_checkpoints
+.ipynb_checkpoints

.gradient/automated-test.sh

@@ -10,7 +10,7 @@
 # 3: Version ID
 # 4: Either the runtime in which we are running or 'upload-reports'
 # 5: Folder in which to save/look for tar.gz report archives
-# 6: Examples utils spec file to process and benchmark 
+# 6: Examples utils spec file to process and benchmark
 # 7: Huggingface token
 # @:8 other arguments are passed to the `examples_utils platform_assesment` command
 # @:9 other arguments are passed to the `examples_utils platform_assesment` command

.gradient/available_ipus.py

@@ -18,4 +18,4 @@
         f" the Notebook ID: {nb_id}"
     ) from err
 # to be captured as a variable in the bash script that calls this python script
-print(num_ipus)
+print(num_ipus)

.gradient/notebook-tests.yaml

@@ -11,47 +11,41 @@ pyg-cluster-GCN:
   notebook:
     file: cluster_gcn_training.ipynb
     timeout: 1200
-  requirements_file: requirements.txt
 
 pyg-message-passing:
   location: ../graph-prediction/gin-molecular-prediction
   generated: true
   notebook:
     file: gin_training.ipynb
     timeout: 1200
-  requirements_file: requirements.txt
 
 pyg-schnet:
   location: ../graph-prediction/schnet-molecular-prediction
   generated: true
   notebook:
     file: schnet_training.ipynb
     timeout: 1200
-  requirements_file: requirements.txt
 
 pyg-tgn:
   location: ../link-prediction/temporal-graph-networks
   generated: true
   notebook:
     file: Train_TGN.ipynb
     timeout: 1200
-  requirements_file: requirements.txt
 
 pyg-nbfnet:
   location: ../link-prediction/NBFNet-for-knowledge-graphs
   generated: true
   notebook:
     file: NBFNet_training.ipynb
-    timeout: 1200
-  requirements_file: requirements.txt
+    timeout: 2400
 
 pyg-ipu-speed-advantage:
   location: ../ipu-speed-advantage
   generated: true
   notebook:
     file: benchmarking_gnn_on_ipus.ipynb
     timeout: 1200
-  requirements_file: requirements.txt
 
 # tutorials
 
@@ -61,46 +55,76 @@ pyg-tut1:
   notebook:
     file: 1_at_a_glance.ipynb
     timeout: 1200
-  requirements_file: requirements.txt
 
 pyg-tut2:
   location: ../learning-pytorch-geometric-on-ipus
   generated: true
   notebook:
     file: 2_a_worked_example.ipynb
     timeout: 1200
-  requirements_file: requirements.txt
 
 pyg-tut3:
   location: ../learning-pytorch-geometric-on-ipus
   generated: true
   notebook:
     file: 3_small_graph_batching_with_padding.ipynb
     timeout: 1200
-  requirements_file: requirements.txt
 
 pyg-tut4:
   location: ../learning-pytorch-geometric-on-ipus
   generated: true
   notebook:
     file: 4_small_graph_batching_with_packing.ipynb
     timeout: 1200
-  requirements_file: requirements.txt
+
+pyg-tut5:
+  location: ../learning-pytorch-geometric-on-ipus
+  generated: true
+  notebook:
+    file: 5_large_graph_sampling.ipynb
+    timeout: 1200
+
+pyg-tut6:
+  location: ../learning-pytorch-geometric-on-ipus
+  generated: true
+  notebook:
+    file: 6_heterogeneous_graphs.ipynb
+    timeout: 1200
+
 
 # MolFeat
 molfeat:
-  location: ../molfeat
+  location: ../archive/molfeat
   generated: true
   notebook:
     file: pytorch_geometric_molfeat.ipynb
     timeout: 1200
-  requirements_file: requirements.txt
 
 # Graphium
 graphium:
-  location: ../graphium
+  location: ../archive/graphium
   generated: true
   notebook:
     file: running-multitask-ipu.ipynb
     timeout: 1200
-  requirements_file: requirements.txt
+
+useful-managing-ipu-resources:
+  location: ../useful-tips
+  generated: true
+  notebook:
+    file: managing_ipu_resources.ipynb
+    timeout: 1000
+
+fraud-detection-1:
+  location: ../fraud-detection
+  generated: true
+  notebook:
+    file: 1_dataset_preprocessing.ipynb
+    timeout: 1800
+
+fraud-detection-2:
+  location: ../fraud-detection
+  generated: true
+  notebook:
+    file: 2_training.ipynb
+    timeout: 1800

.gradient/prepare-datasets.sh

@@ -21,4 +21,4 @@ if [[ "${1:-}" == 'test' ]]; then
 elif [[ "${2:-}" == 'test' ]]; then
     #source .gradient/automated-test.sh "${@:2}"
     /notebooks/.gradient/automated-test.sh $3 $4 $5 $6 $7 $8 "${@:9}"
-fi
+fi