feat: reorganize documentation (#330)

* feat: reorganize documentation

Signed-off-by: SdgJlbl <sarah.diot-girard@owkin.com>

* chore: move examples files

Signed-off-by: SdgJlbl <sarah.diot-girard@owkin.com>

* chore: reorganise how-to guides

Signed-off-by: SdgJlbl <sarah.diot-girard@owkin.com>

* fix: update .gitignore

Signed-off-by: SdgJlbl <sarah.diot-girard@owkin.com>

* fix: Binder

Signed-off-by: SdgJlbl <sarah.diot-girard@owkin.com>

* fix: links in examples

Signed-off-by: SdgJlbl <sarah.diot-girard@owkin.com>

* fix: diabetes example

Signed-off-by: SdgJlbl <sarah.diot-girard@owkin.com>

* fix: reorganise deployments how-to

Signed-off-by: SdgJlbl <sarah.diot-girard@owkin.com>

* fix: cyclic example

Signed-off-by: SdgJlbl <sarah.diot-girard@owkin.com>

* fix: renaming things and ordering

Signed-off-by: SdgJlbl <sarah.diot-girard@owkin.com>

---------

Signed-off-by: SdgJlbl <sarah.diot-girard@owkin.com>

.binder/postBuild

@@ -9,11 +9,7 @@ pip uninstall -y -r docs/requirements.txt
 
 # move examples to the notebooks folder
 mv docs/build/html/notebooks .
-mv examples/titanic_example/assets notebooks/auto_examples/titanic_example
-mv examples/diabetes_example/assets notebooks/auto_examples/diabetes_example
-mv substrafl_examples/get_started/torch_fedavg_assets notebooks/substrafl_doc/examples/get_started
-mv substrafl_examples/go_further/diabetes_substrafl_assets notebooks/substrafl_doc/examples/go_further
-mv substrafl_examples/go_further/sklearn_fedavg_assets notebooks/substrafl_doc/examples/go_further
+mv examples notebooks/examples
 
 # delete everything but the notebooks folder and the substra dependencies
 shopt -s extglob

.gitignore

@@ -135,10 +135,10 @@ _build/
 # Misc build artefacts
 tmp/**
 
-docs/source/auto_examples/**
+docs/source/examples/**
+doc/source/substrafl_doc/examples/**
 docs/source/documentation/references/**
 docs/source/substrafl_doc/api
-docs/source/substrafl_doc/examples
 
 # Zip files
 *.zip

docs/Makefile

@@ -7,7 +7,7 @@ SPHINXOPTS    ?= -W --keep-going -n
 SPHINXBUILD   ?= sphinx-build
 SOURCEDIR     = source
 BUILDDIR      = build
-SUBSTRAEXAMPLEDIR 		= source/auto_examples
+SUBSTRAEXAMPLEDIR 		= source/examples/substra_core
 SUBSTRADOCDIR 		= source/documentation/references
 SUBSTRAFLEXAMPLEDIR	= source/substrafl_doc/examples
 SUBSTRAFLDOCDIR	= source/substrafl_doc/api

docs/source/conf.py

@@ -111,29 +111,29 @@ def zip_dir(source_dir, zip_file_name):
                 )
 
 
-assets_dir_titanic = Path(__file__).parents[2] / "examples" / "titanic_example" / "assets"
+assets_dir_titanic = Path(__file__).parents[2] / "examples" / "substra_core" / "titanic_example" / "assets"
 zip_dir(assets_dir_titanic, "titanic_assets.zip")
 
-assets_dir_diabetes = Path(__file__).parents[2] / "examples" / "diabetes_example" / "assets"
+assets_dir_diabetes = Path(__file__).parents[2] / "examples" / "substra_core" / "diabetes_example" / "assets"
 zip_dir(assets_dir_diabetes, "diabetes_assets.zip")
 
 assets_dir_substrafl_torch_fedavg = (
-    Path(__file__).parents[2] / "substrafl_examples" / "get_started" / "torch_fedavg_assets"
+    Path(__file__).parents[2] / "examples" / "substrafl" / "get_started" / "torch_fedavg_assets"
 )
 zip_dir(assets_dir_substrafl_torch_fedavg, "torch_fedavg_assets.zip")
 
 assets_dir_substrafl_diabetes = (
-    Path(__file__).parents[2] / "substrafl_examples" / "go_further" / "diabetes_substrafl_assets"
+    Path(__file__).parents[2] / "examples" / "substrafl" / "go_further" / "diabetes_substrafl_assets"
 )
 zip_dir(assets_dir_substrafl_diabetes, "diabetes_substrafl_assets.zip")
 
 assets_dir_substrafl_sklearn_fedavg = (
-    Path(__file__).parents[2] / "substrafl_examples" / "go_further" / "sklearn_fedavg_assets"
+    Path(__file__).parents[2] / "examples" / "substrafl" / "go_further" / "sklearn_fedavg_assets"
 )
 zip_dir(assets_dir_substrafl_sklearn_fedavg, "sklearn_fedavg_assets.zip")
 
 assets_dir_substrafl_sklearn_fedavg = (
-    Path(__file__).parents[2] / "substrafl_examples" / "go_further" / "torch_cyclic_assets"
+    Path(__file__).parents[2] / "examples" / "substrafl" / "go_further" / "torch_cyclic_assets"
 )
 zip_dir(assets_dir_substrafl_sklearn_fedavg, "torch_cyclic_assets.zip")
 
@@ -418,14 +418,14 @@ def reformat_md_section_links(file_path: Path):
     "remove_config_comments": True,
     "doc_module": "substra",
     "reference_url": {"Substra": None},
-    "examples_dirs": ["../../examples", "../../substrafl_examples"],
-    "gallery_dirs": ["auto_examples", "substrafl_doc/examples"],
+    "examples_dirs": ["../../examples/substra_core", "../../examples/substrafl"],
+    "gallery_dirs": ["examples/substra_core", "examples/substrafl"],
     "subsection_order": ExplicitOrder(
         [
-            "../../examples/titanic_example",
-            "../../examples/diabetes_example",
-            "../../substrafl_examples/get_started",
-            "../../substrafl_examples/go_further",
+            "../../examples/substra_core/titanic_example",
+            "../../examples/substra_core/diabetes_example",
+            "../../examples/substrafl/get_started",
+            "../../examples/substrafl/go_further",
         ]
     ),
     "download_all_examples": False,

docs/source/documentation/api_reference.rst

@@ -1,5 +1,5 @@
-API reference
-=============
+Substra API reference
+=====================
 
 `substra` version: |substra_version|
 

docs/source/documentation/concepts.rst

@@ -6,6 +6,11 @@ Main Substra concepts
 .. contents::
     :depth: 3
 
+.. toctree::
+   :maxdepth: 1
+
+   substra_tools
+
 Assets
 ------
 

docs/source/operations/howto/existing-volumes.rst → docs/source/how-to/deploying-substra/howto/existing-volumes.rst

@@ -1,6 +1,6 @@
-********************
-Use existing volumes
-********************
+***************************
+How-to use existing volumes
+***************************
 
 By default, Substra instanciates PersistentVolumeClaims (PVCs) on the fly, which is generally convenient; but this might not suit your needs, in which case Substra can also use preexisting PVCs rather than make new ones.
 

docs/source/operations/howto/external-database.rst → docs/source/how-to/deploying-substra/howto/external-database.rst

@@ -1,8 +1,8 @@
 .. _ops howto external database:
 
-************************
-Use an external database
-************************
+*******************************
+How-to use an external database
+*******************************
 
 By default, Substra components use their own integrated postgres databases (one per backend, and one for the orchestrator in centralized mode).
 

docs/source/operations/howto/sso-oidc.rst → docs/source/how-to/deploying-substra/howto/sso-oidc.rst

@@ -1,6 +1,6 @@
-******************************
-Set up SSO with OpenID Connect
-******************************
+*************************************
+How-to set up SSO with OpenID Connect
+*************************************
 
 Substra supports external user management through OpenID Connect (OIDC). It is done per-backend, so each organization can use their own user provider and cohabit on the same network.
 

docs/source/operations/overview.rst → docs/source/how-to/deploying-substra/index.rst

@@ -1,20 +1,32 @@
-********
-Overview
-********
+How-to guides for deploying Substra
+===================================
 
 This section is of concern if you are **Deploying Substra in production**.
 
-- :ref:`ops walkthrough` guides you through deploying a production environment
-- :ref:`ops howto` are guides on some tasks of interest.
-- :ref:`ops upgrade notes` should be read when upgrading from one version to the next.
-- :ref:`compatibility table` holds a reference of Substra versions.
+Familiarity with infrastructure, and Kubernetes in particular, is recommended.
+
+
+:ref:`The walkthrough guide <ops walkthrough>` takes you step by step through deploying a production environment.
+More specific how-to guides cover additional points.
+
+:ref:`ops upgrade notes` cover relevant changes when upgrading from one version to the next.
+
+The :ref:`compatibility table` contains a reference of Substra versions compatible with one another.
+
+.. toctree::
+   :maxdepth: 2
+   :hidden:
+
+   walkthrough.rst
+   howto/existing-volumes.rst
+   howto/external-database.rst
+   howto/sso-oidc.rst
+   upgrade-notes.rst
 
-How Substra is deployed
-=======================
 
 Substra is meant to be deployed as part of a federated learning network. Each participant *organization* will set up their own *Substra node*, from which their users can connect to the network and run machine learning algorithms on the data registered by participant on their own node.
 
-.. image:: ../static/schemes/stack-technical-scheme.svg
+.. image:: ../../static/schemes/stack-technical-scheme.svg
   :width: 800
   :align: center
   :alt: Substra Components Scheme
@@ -40,3 +52,4 @@ The orchestrator needs the following resources:
 * 4 CPU
 * 16 GB of RAM
 * 100 GB of storage
+

docs/source/operations/walkthrough.rst → docs/source/how-to/deploying-substra/walkthrough.rst

@@ -1,8 +1,8 @@
 .. _ops walkthrough:
 
-***********
-Walkthrough
-***********
+*********************
+How-to deploy Substra
+*********************
 
 This section will guide you through a production deployment with two Substra nodes.
 

docs/source/operations/walkthrough/10-prerequisites.rst → docs/source/how-to/deploying-substra/walkthrough/10-prerequisites.rst

@@ -1,6 +1,6 @@
-**********************
-Consider Prerequisites
-**********************
+*******************
+Check Prerequisites
+*******************
 
 Substra version
 ===============

docs/source/how-to/developing-substra/index.rst

@@ -0,0 +1,9 @@
+How-to guides for developing Substra
+====================================
+
+The following guides might be of interest if you are interested in contributing to Substra.
+
+.. toctree::
+   :maxdepth: 1
+
+   local-deployment.rst

docs/source/contributing/local-deployment.rst → docs/source/how-to/developing-substra/local-deployment.rst

@@ -192,7 +192,7 @@ Launching
 Launching computations
 ======================
 
-One way to test that everything is working fine is to launch computations on your local deployment. To do that you can use the :ref:`MNIST federated learning example <substrafl_doc/examples/get_started/run_mnist_torch:Using Torch FedAvg on MNIST dataset>` and setup the clients with the following values:
+One way to test that everything is working fine is to launch computations on your local deployment. To do that you can use the :ref:`MNIST federated learning example <examples/substrafl/get_started/run_mnist_torch:Using Torch FedAvg on MNIST dataset>` and setup the clients with the following values:
 
    .. code-block:: python
 
@@ -239,9 +239,9 @@ If this command fails and you still have pods up, you can use the following comm
 Next steps
 ==========
 
-Now you are ready to go, you can either run the :doc:`Substra examples </auto_examples/index>` or the :doc:`SubstraFL examples </substrafl_doc/examples/index>`.
+Now you are ready to go, you can either run the :doc:`Substra examples </examples/substra_core/index>` or the :doc:`SubstraFL examples </examples/substrafl/index>`.
 
-This local deployment is for developing or testing Substra. If you want to have a more production-ready deployment and a more customized set-up, have a look at the :ref:`deployment section <operations/overview:Overview>`.
+This local deployment is for developing or testing Substra. If you want to have a more production-ready deployment and a more customized set-up, have a look at the :doc:`deployment section <../deploying-substra/index>`.
 
 Documentation on running tests on any of the Substra components is available on the component repositories, see `substra <https://github.com/substra/substra>`_, `substrafl <https://github.com/substra/substrafl>`_, `substra-tools <https://github.com/substra/substra-tools>`_, substra-backend_, orchestrator_, substra-frontend_ and `substra-tests <https://github.com/substra/substra-tests>`_ repositories.
 
@@ -256,7 +256,7 @@ Troubleshooting
       * if you are using a release you can use :ref:`the compatibility table <additional/release:Compatibility table>`.
       * if you are using the latest commit from the ``main`` git branch, check that you are up-to-date and see if there were any open issue in the repositories or any bugfixes in the latest commits.
 
-   You can also go through :doc:`the instructions one more time </contributing/local-deployment>`, maybe they changed since you last saw them.
+   You can also go through :doc:`the instructions one more time </how-to/developing-substra/local-deployment>`, maybe they changed since you last saw them.
 
 Troubleshooting prerequisites
 -----------------------------

docs/source/how-to/using-substra/index.rst

@@ -0,0 +1,21 @@
+How-to guides for using Substra
+===============================
+
+The following guides might be of interest if you are a data scientist using Substra.
+
+These are made to help on specific points, if you wish to have a general walkthrough on how to work with Substra,
+you can check the :doc:`tutorials </examples/substrafl/index>`.
+
+.. toctree::
+   :maxdepth: 1
+
+
+   debug.rst
+   get_performances.rst
+   client_configuration.rst
+   api_tokens_generation.rst
+   gpu.rst
+   r_scripts.rst
+
+
+

docs/source/documentation/r_scripts.rst → docs/source/how-to/using-substra/r_scripts.rst

@@ -6,7 +6,7 @@ to accommodate running tasks in other programming languages.
 This how-to guide explains how to run scripts written in R with Substra.
 This uses the low-level interface of Substra and requires writing more boilerplate code than using the high-level interface of SubstraFL.
 If you are not familiar with the Substra low-level library, you should read the
-:doc:`Substra introductory example </auto_examples/titanic_example/run_titanic>` first.
+:doc:`Substra introductory example </examples/substra_core/titanic_example/run_titanic>` first.
 
 .. caution:: This guide provides an easy to run some scripts in another language.
     The scripts are wrapped up in a Python process, so performances might be limited.

docs/source/index.rst

@@ -46,8 +46,8 @@ Installation
 
 **Server side**: There are two options to deploy the server side of Substra (backend, frontend and orchestrator):
 
-* :ref:`Local deployment <contributing/local-deployment:Local deployment>`: to deploy locally on a single one machine. Useful for quick tests and for development.
-* :ref:`Production deployment <operations/Overview:Overview>`: for real deployments.
+* :ref:`Local deployment <how-to/developing-substra/local-deployment:Local deployment>`: to deploy locally on a single one machine. Useful for quick tests and for development.
+* :doc:`Production deployment <how-to/deploying-substra/index>`: for real deployments.
 
 .. note::
    You can start doing local FL experiments with Substra by installing only the **client side**.
@@ -57,59 +57,66 @@ Links
 
 Some quick links:
 
-* :ref:`MNIST federated learning example <substrafl_doc/examples/get_started/run_mnist_torch:Using Torch FedAvg on MNIST dataset>`
-* :ref:`SubstraFL overview <substrafl_doc/substrafl_overview:Overview>`
+* :ref:`MNIST federated learning example <examples/substrafl/get_started/run_mnist_torch:Using Torch FedAvg on MNIST dataset>`
+* :doc:`SubstraFL overview <substrafl_doc/substrafl_overview>`
 * :ref:`Compatibility table <additional/release:Compatibility table>`
-* :ref:`How to deploy Substra for Site Reliability Engineers <operations/Overview:Overview>`
+* :doc:`How to deploy Substra for Site Reliability Engineers <how-to/deploying-substra/index>`
 * :ref:`Community <additional/community:Community>`
 * `Subscribe to our newsletter <https://lists.lfaidata.foundation/g/substra-announce/join>`_
 
 
 .. toctree::
    :glob:
    :maxdepth: 2
-   :caption: Substrafl documentation
+   :caption: What is Substra
    :hidden:
 
 
-   substrafl_doc/index.rst
+   substrafl_doc/substrafl_overview
+   documentation/concepts
+   documentation/components
+
 
 
 .. toctree::
    :glob:
    :maxdepth: 2
-   :caption: Substra documentation
+   :caption: Tutorials
    :hidden:
 
-   documentation/user_guide.rst
-   auto_examples/index
-   documentation/api_reference.rst
+   examples/substrafl/index
+   examples/substra_core/index
 
 
 .. toctree::
    :glob:
    :maxdepth: 2
-   :caption: Contributing to Substra
+   :caption: API Reference
    :hidden:
 
-   contributing/contributing-guide.rst
-   contributing/code-of-conduct.rst
-   contributing/components.rst
-   contributing/local-deployment.rst
+   substrafl_doc/api/index.rst
+   documentation/api_reference.rst
 
 
 .. toctree::
    :glob:
    :maxdepth: 1
-   :caption: Deploying Substra
+   :caption: How-to guides
    :hidden:
 
-   operations/overview.rst
-   operations/walkthrough.rst
-   operations/howto.rst
-   operations/upgrade-notes.rst
+   how-to/using-substra/index.rst
+   how-to/deploying-substra/index.rst
+   how-to/developing-substra/index.rst
+
 
+.. toctree::
+   :glob:
+   :maxdepth: 1
+   :caption: Contributing guide
+   :hidden:
 
+   contributing/contributing-guide.rst
+   contributing/code-of-conduct.rst
 
 .. toctree::
    :glob: