Merge branch 'packinfo-docs' into 'master'

Improve documentation of GPU code

See merge request walberla/walberla!672

apps/tutorials/basics/01_BlocksAndFields.dox

@@ -6,7 +6,7 @@ namespace walberla {
 \brief Introduction to block structure and field.
 
 This tutorial walks you through the process of creating a simple waLBerla application. 
-The source file of this tutorial can be found in apps/tutorials/01_BlocksAndFields.cpp.
+The source file of this tutorial can be found in `apps/tutorials/01_BlocksAndFields.cpp`.
 To compile and run this example, go to your build directory into `apps/tutorials` type `make`
 and run the generated executable.
 
@@ -153,7 +153,7 @@ Using this setup mechanism, waLBerla does not enforce that the fields have the s
 
 Remember: For waLBerla, a block is just a container for arbitrary data - and a field is just an "arbitrary" data item stored on each block.
 Block data does not have to be any waLBerla data structure. It is possible to store any type of data on a block, 
-so instead of using the field class, we could, for example, have used a std::vector<std::vector<double> > to store our lattice.
+so instead of using the field class, we could, for example, have used a `std::vector<std::vector<double>>` to store our lattice.
 
 The callback function can now be registered at the block storage with the following piece of code:
 
@@ -171,7 +171,7 @@ dock widget can then be used to display slices of the field.
 
 \image html tutorial_basics01_field.jpeg
 
-The next tutorial contains the writing of algorithms operating on block data: \ref tutorial02
+The next tutorial contains the writing of algorithms operating on block data: \ref tutorial_basics_02
 
 \tableofcontents
 

apps/tutorials/gpu/01_GameOfLife_cuda.dox

@@ -37,9 +37,9 @@ auto hostFieldAllocator = make_shared< gpu::HostFieldAllocator<real_t> >();
 BlockDataID const cpuFieldID =field::addToStorage< ScalarField >(blocks, "CPU Field", real_c(0.0), field::fzyx, uint_c(1), hostFieldAllocator);
 \endcode
 
-Now we initialize the CPU field just like in the previous tutorial \ref tutorial_basics03 .
+Now we initialize the CPU field just like in the previous tutorial \ref tutorial_basics_03 .
 Then two GPU fields are created: "source" and "destination" field. The helper function
-gpu::addGPUFieldToStorage() creates a gpu::GPUField field of the same size and layout of the given
+\ref gpu::addGPUFieldToStorage() creates a \ref gpu::GPUField field of the same size and layout of the given
 CPU field:
 \code
 BlockDataID gpuFieldSrcID = gpu::addGPUFieldToStorage<ScalarField>( blocks, cpuFieldID, "GPU Field Src" );
@@ -118,10 +118,10 @@ Note that copying data is costly and thus we don't want to do this in every time
 
 \section gpu01_comm Communication
 
-For this tutorial we use the gpu::communication::UniformGPUScheme that first collects all data in a buffer and
-sends only one message per communication step and neighbor. For the PackInfo we use the MemcpyPackInfo. It receives
-a buffer located on the GPU and fills it using memcpy operations
-If the GPU library is build with MPI support this buffer can be send to other GPUs without a copy to the CPU.
+For this tutorial we use the \ref gpu::communication::UniformGPUScheme that first collects all data in a buffer and
+sends only one message per communication step and neighbor. For the `PackInfo` we use the \ref gpu::communication::MemcpyPackInfo.
+It receives a buffer located on the GPU and fills it using memcpy operations.
+If the GPU library is built with MPI support this buffer can be sent to other GPUs without a copy to the CPU.
 Otherwise the copying will be done in the back by the communication class.
 
 \code

src/blockforest/communication/NonUniformPackInfo.h

@@ -52,28 +52,28 @@ class NonUniformPackInfo
    /**
     * Should return true if the amount of data that is packed for a given block in direction
     * "dir" is guaranteed to remain constant over time. False otherwise.
-    * If you are not sure what to return, return false! Returning false is always save.
+    * If you are not sure what to return, return false! Returning false is always safe.
     * Falsely return true will lead to errors! However, if the data can be guaranteed to remain
     * constant over time, returning true enables performance optimizations during the communication.
     */
    virtual bool constantDataExchange() const = 0;
 
    /**
-    * Must return false if calling unpackData and/or communicateLocal is not thread-safe.
+    * Must return false if calling `unpackData*()` and/or `communicateLocal*()` methods is not thread-safe.
     * True otherwise.
-    * If you are not sure what to return, return false! Returning false is always save.
-    * Falsely return true will most likely lead to errors! However, if both unpackData AND
-    * communicateLocal are thread-safe, returning true can lead to performance improvements.
+    * If you are not sure what to return, return false! Returning false is always safe.
+    * Falsely return true will most likely lead to errors! However, if both `unpackData*()` AND
+    * `communicateLocal*()` are thread-safe, returning true can lead to performance improvements.
     */
    virtual bool threadsafeReceiving() const = 0;
 
-   /// Must be thread-safe! Calls packDataImpl.
+   /// Must be thread-safe! Calls \ref packDataEqualLevelImpl.
    inline void packDataEqualLevel( const Block * sender, stencil::Direction dir, mpi::SendBuffer & buffer ) const;
 
-   /// If NOT thread-safe, threadsafeReceiving must return false!
+   /// If NOT thread-safe, \ref threadsafeReceiving must return false!
    virtual void unpackDataEqualLevel( Block * receiver, stencil::Direction dir, mpi::RecvBuffer & buffer ) = 0;
 
-   /// If NOT thread-safe, threadsafeReceiving must return false!
+   /// If NOT thread-safe, \ref threadsafeReceiving must return false!
    virtual void communicateLocalEqualLevel( const Block * sender, Block * receiver, stencil::Direction dir ) = 0;
 
    inline  void packDataCoarseToFine        ( const Block * coarseSender, const BlockID & fineReceiver, stencil::Direction dir, mpi::SendBuffer & buffer ) const;

src/blockforest/communication/UniformToNonUniformPackInfoAdapter.h

@@ -32,7 +32,7 @@ namespace communication {
 
 //*******************************************************************************************************************
 /*! 
- * Adapter to use a UniformPackInfo in a NonUniformBufferedScheme. No communication between coarse <-> fine blocks
+ * Adapter to use a \ref communication::UniformPackInfo in a \ref NonUniformBufferedScheme. No communication between coarse <-> fine blocks
  * happens.
  */
 //*******************************************************************************************************************
@@ -51,25 +51,25 @@ class UniformToNonUniformPackInfoAdapter : public NonUniformPackInfo
    /**
    * Should return true if the amount of data that is packed for a given block in direction
    * "dir" is guaranteed to remain constant over time. False otherwise.
-   * If you are not sure what to return, return false! Returning false is always save.
+   * If you are not sure what to return, return false! Returning false is always safe.
    * Falsely return true will lead to errors! However, if the data can be guaranteed to remain
    * constant over time, returning true enables performance optimizations during the communication.
    */
    virtual bool constantDataExchange() const { return uniformPackInfo_->constantDataExchange(); }
 
    /**
-   * Must return false if calling unpackData and/or communicateLocal is not thread-safe.
+   * Must return false if calling `unpackData*()` and/or `communicateLocal*()` methods is not thread-safe.
    * True otherwise.
-   * If you are not sure what to return, return false! Returning false is always save.
-   * Falsely return true will most likely lead to errors! However, if both unpackData AND
-   * communicateLocal are thread-safe, returning true can lead to performance improvements.
+   * If you are not sure what to return, return false! Returning false is always safe.
+   * Falsely return true will most likely lead to errors! However, if both `unpackData*()` AND
+   * `communicateLocal*()` are thread-safe, returning true can lead to performance improvements.
    */
    virtual bool threadsafeReceiving() const { return uniformPackInfo_->threadsafeReceiving(); }
 
-   /// If NOT thread-safe, threadsafeReceiving must return false!
+   /// If NOT thread-safe, \ref threadsafeReceiving must return false!
    virtual void unpackDataEqualLevel( Block * receiver, stencil::Direction dir, mpi::RecvBuffer & buffer ) { uniformPackInfo_->unpackData( receiver, dir, buffer ); }
 
-   /// If NOT thread-safe, threadsafeReceiving must return false!
+   /// If NOT thread-safe, \ref threadsafeReceiving must return false!
    virtual void communicateLocalEqualLevel( const Block * sender, Block * receiver, stencil::Direction dir ) { uniformPackInfo_->communicateLocal( sender, receiver, dir ); }
 
    virtual void unpackDataCoarseToFine( Block * /*fineReceiver*/, const BlockID & /*coarseSender*/, stencil::Direction /*dir*/, mpi::RecvBuffer & /*buffer*/ ) { }

src/communication/UniformPackInfo.h

@@ -35,18 +35,21 @@ namespace communication {
 
 
 /**
- * \brief UniformPackInfo encapsulates information on how to extract data from blocks,
- * that should be communicated (see packData() ) to neighboring blocks
- * and how to inject this data in a receiving block (see unpackData() )
+ * \brief Data packing/unpacking for ghost layer based communication of a field.
  *
- * Another special method exists for communication between two blocks,
- * which are allocated on the same
- * process. In this case the data does not have be communicated via a buffer,
+ * Encapsulate information on how to extract data from blocks that should be
+ * communicated to neighboring blocks (see \ref packData())
+ * and how to inject this data in a receiving block (see \ref unpackData()).
+ * This involves a memory buffer and two memory copy operations.
+ *
+ * A special method exists for communication between two blocks which are
+ * allocated on the same process (see \ref communicateLocal()).
+ * In this case the data does not have be communicated via a buffer,
  * but can be copied directly.
  *
  * Data that is packed in direction "dir" at one block is unpacked in
  * direction "stencil::inverseDir[dir]" at the neighboring block. This
- * behavior must be implemented in "communicateLocal"!
+ * behavior must be implemented in \ref communicateLocal()!
  *
  * \ingroup communication
  */
@@ -65,23 +68,25 @@ class UniformPackInfo
    /**
     * Should return true if the amount of data that is packed for a given block in direction
     * "dir" is guaranteed to remain constant over time. False otherwise.
-    * If you are not sure what to return, return false! Returning false is always save.
-    * Falsely return true will lead to errors! However, if the data can be guaranteed to remain
+    * If you are not sure what to return, return false! Returning false is always safe.
+    * Falsely returning true will lead to errors! However, if the data can be guaranteed to remain
     * constant over time, returning true enables performance optimizations during the communication.
     */
    virtual bool constantDataExchange() const = 0;
 
    /**
-    * Must return false if calling unpackData and/or communicateLocal is not thread-safe.
+    * Must return false if calling \ref unpackData and/or \ref communicateLocal is not thread-safe.
     * True otherwise.
-    * If you are not sure what to return, return false! Returning false is always save.
-    * Falsely return true will most likely lead to errors! However, if both unpackData AND
-    * communicateLocal are thread-safe, returning true can lead to performance improvements.
+    * If you are not sure what to return, return false! Returning false is always safe.
+    * Falsely returning true will most likely lead to errors! However, if both \ref unpackData AND
+    * \ref communicateLocal are thread-safe, returning true can lead to performance improvements.
     */
    virtual bool threadsafeReceiving() const = 0;
 
    /**
-    * Packs data from a block into a send buffer. Must be thread-safe! Calls packDataImpl.
+    * \brief Pack data from a block into a send buffer.
+    *
+    * Must be thread-safe! Calls \ref packDataImpl.
     *
     * @param sender     the block whose data should be packed into a buffer
     * @param dir        pack data for neighbor in this direction
@@ -91,19 +96,21 @@ class UniformPackInfo
    inline void packData( const IBlock * sender, stencil::Direction dir, mpi::SendBuffer & buffer ) const;
 
    /**
-    * Unpacks received Data.
-    * If NOT thread-safe, threadsafeReceiving must return false!
+    * \brief Unpack received Data.
+    *
+    * If NOT thread-safe, \ref threadsafeReceiving must return false!
     *
     * @param receiver the block where the unpacked data should be stored into
     * @param dir      receive data from neighbor in this direction
-    * @param buffer
+    * @param buffer   buffer for reading the data from
     */
    virtual void unpackData( IBlock * receiver, stencil::Direction dir, mpi::RecvBuffer & buffer ) = 0;
 
    /**
-    * Function to copy data from one local block to another local block.
+    * \brief Copy data from one local block to another local block.
+    *
     * Both blocks are allocated on the current process.
-    * If NOT thread-safe, threadsafeReceiving must return false!
+    * If NOT thread-safe, \ref threadsafeReceiving must return false!
     *
     * @param sender    id of block where the data should be copied from
     * @param receiver  id of block where the data should be copied to
@@ -134,7 +141,9 @@ class UniformPackInfo
 protected:
 
    /**
-    * Packs data from a block into a send buffer. Must be thread-safe!
+    * \brief Pack data from a block into a send buffer.
+    *
+    * Must be thread-safe!
     *
     * @param sender     the block whose data should be packed into a buffer
     * @param dir        pack data for neighbor in this direction

src/gpu/FieldAccessor.h

@@ -31,6 +31,13 @@ namespace gpu
 
 
 
+   /**
+    * \brief Handle to the underlying device data of a \ref GPUField.
+    *
+    * Encapsulate the device memory pointer and offsets necessary
+    * to calculate the address of a cell from a GPU kernel's thread
+    * coordinates in the thread block.
+    */
    template<typename T>
    class FieldAccessor
    {

src/gpu/FieldIndexing.h

@@ -44,6 +44,14 @@ namespace gpu
 template< typename T >
 class GPUField;
 
+/**
+ * \brief Utility class to generate handles to the underlying device data of a \ref GPUField.
+ *
+ * Pre-calculate memory offsets of a \ref GPUField for a given slice,
+ * cell interval, or the entire grid with or without the ghost layer,
+ * and store them in a \ref FieldAccessor handle.
+ * That handle is obtained by calling \ref gpuAccess().
+ */
 template< typename T >
 class FieldIndexing
 {

src/gpu/GPUField.h

@@ -45,16 +45,20 @@ namespace gpu
    *  Basically a wrapper around a CUDA/HIP device pointer together with size information about the field
    *  i.e. sizes in x,y,z,f directions and number of ghost layers.
    *
-   *  Internally represented by a gpuPitchedPtr which is allocated with gpuMalloc3D to take padding of the
-   *  innermost coordinate into account.
+   *  Internally represented by a \c gpuPitchedPtr which is allocated with extra padding for the
+   *  innermost coordinate.
+   *  Pitched memory is a type of non-linear memory where padding is introduced
+   *  to optimize data alignment and thus reduce data access latency,
+   *  for example by avoiding shared memory bank conflicts.
    *
    *  Supports Array-of-Structures (AoS,zyxf) layout and Structure-of-Arrays (SoA, fzyx) layout, in a similar way
-   *  to field::Field
+   *  to \ref field::Field
    *
-   *  To work with the GPUField look at the gpu::fieldCpy functions to transfer a field::Field to a gpu::GPUField
+   *  To work with the \ref gpu::GPUField look at the \ref gpu::fieldCpy functions to transfer a \ref field::Field to a \ref gpu::GPUField
    *  and vice versa.
-   *  When writing device kernels for GPUFields have a look at the FieldIndexing and FieldAccessor concepts.
-   *  These simplify the "iteration" i.e. indexing of cells in GPUFields.
+   *
+   *  When writing device kernels for a \ref GPUField, have a look at the \ref FieldIndexing and \ref FieldAccessor concepts.
+   *  These simplify the "iteration" i.e. indexing of cells in a \ref GPUField.
    */
    //*******************************************************************************************************************
    template<typename T>

src/gpu/GPURAII.h

@@ -13,7 +13,7 @@
 //  You should have received a copy of the GNU General Public License along
 //  with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
 //
-//! \file CudaRAII.h
+//! \file GPURAII.h
 //! \ingroup gpu
 //! \author Martin Bauer <martin.bauer@fau.de>
 //

src/gpu/communication/GPUPackInfo.h

@@ -47,10 +47,28 @@ namespace walberla::gpu::communication {
 
 
 /**
- * Data packing/unpacking for ghost layer based communication of a gpu::GPUField
+ * \brief Data packing/unpacking for ghost layer based communication of a \ref GPUField.
+ *
+ * Encapsulate information on how to extract data from blocks that should be
+ * communicated to neighboring blocks (see \ref packDataImpl())
+ * and how to inject this data in a receiving block (see \ref unpackData()).
+ * This involves a host memory buffer and two device-to-host memory copy operations.
+ *
+ * A special method exists for communication between two blocks which are
+ * allocated on the same process (see \ref communicateLocal()).
+ * In this case the data does not have be communicated via a host buffer,
+ * but can be sent directly. This involves a single device-to-device memory
+ * copy operation.
+ *
+ * Data that is packed in direction "dir" at one block is unpacked in
+ * direction "stencil::inverseDir[dir]" at the neighboring block.
+ * This behavior must be implemented in \ref communicateLocal()!
+ *
+ * See \ref MemcpyPackInfo for a more efficient packing/unpacking method
+ * where the buffer is stored in device memory rather than in host memory.
+ *
  * \ingroup gpu
- * Template Parameters:
- *    - GPUField_T   A fully qualified GPUField.
+ * \tparam GPUField_T   A fully qualified \ref GPUField.
  */
 template<typename GPUField_T>
 class GPUPackInfo : public walberla::communication::UniformPackInfo