249 alp passes 0 as max calls to lpf collectives initialisation (#302)

This MR fixes a bug where `lpf_coll_t` was not initialised properly. This bug never resulted in a noticeable issue, since the present implementation of the LPF collectives ignores the erroneous argument that ALP was passing in. The nevertheless not-up-to-spec initialisation of LPF collectives happened repeatedly throughout the code base. This MR totally overhauls the implementation of the collectives to address the issue.

Within LPF, there are two classes of collectives: the one exposed via the public API `grb::collectives<>`, and raw internal collectives that are used in the implementation of e.g. the `BSP1D` and `hybrid` backends. Prior to this MR, the implementations of these collectives were disjoint, while with this MR, both types of collectives are managed by a shared context (internally encapsulated by the `BSP1D_Data` object). This central management means every process maintains but a single `lpf_coll_t` instance that is reused every time a public or internal collective is called.

The revision of the collectives also now includes a systematic choice where collectives on scalars are orchestrated through the global buffer so as to prevent having to register the scalar memory addresses every time a collective is called. This is likely to cut latency costs significantly compared to the status prior to this MR. For collectives on arrays, the source and/or destination arrays will be registered which incurs a latency costs-- but a cost that, assuming large enough array sizes, is much lower than that of having to copy the payload to and from the global buffer. 

Features this MR introduces in realising the refactored collectives:
- a notion of a final backend, which is the final backend a composed backend relies on. The FinalBackend class collects a set of minimal and low-level functions that all final backends should provide, such as memcpy; and
- a utility that translates LPF error codes into corresponding ALP error codes.

Other issues this MR addresses:
- BSP1D did not seem to handle some border cases of `grb::setElement` correctly -- these would trigger only in out-of-memory conditions, or in erroneous use of the primitive, so impact of the bug is expected to be minor;
- the `grb::setElement` and `grb::select` base specifications did not list all error conditions or did not do so correctly, herewith fixed;
- a metabug in `grb::select` had in `_DEBUG` mode an attempt to print an iterator to `stdout`, instead of first dereferencing it-- herewith fixed.

As always, the MR includes both related and unrelated code style fixes.

Author: anyzelman

include/CMakeLists.txt

@@ -44,15 +44,16 @@ set( root_files
 	"graphblas/backends.hpp" "graphblas/benchmark.hpp"
 	"graphblas/blas0.hpp" "graphblas/blas1.hpp" "graphblas/blas2.hpp"
 	"graphblas/blas3.hpp" "graphblas/collectives.hpp" "graphblas/config.hpp"
-	"graphblas/coordinates.hpp" "graphblas/descriptors.hpp" "graphblas/distribution.hpp"
-	"graphblas/exec.hpp" "graphblas/identities.hpp" "graphblas/init.hpp"
-	"graphblas/internalops.hpp" "graphblas/io.hpp" "graphblas/iomode.hpp"
-	"graphblas/matrix.hpp" "graphblas/monoid.hpp" "graphblas/ops.hpp"
-	"graphblas/phase.hpp" "graphblas/pinnedvector.hpp" "graphblas/properties.hpp"
-	"graphblas/rc.hpp" "graphblas/semiring.hpp" "graphblas/spmd.hpp"
-	"graphblas/tags.hpp" "graphblas/type_traits.hpp" "graphblas/utils.hpp"
-	"graphblas/vector.hpp" "graphblas/synchronizedNonzeroIterator.hpp"
-	"graphblas/nonzeroStorage.hpp" "graphblas/selection_ops.hpp"
+	"graphblas/coordinates.hpp" "graphblas/descriptors.hpp"
+	"graphblas/distribution.hpp" "graphblas/exec.hpp" "graphblas/final.hpp"
+	"graphblas/identities.hpp" "graphblas/init.hpp" "graphblas/internalops.hpp"
+	"graphblas/io.hpp" "graphblas/iomode.hpp" "graphblas/matrix.hpp"
+	"graphblas/monoid.hpp" "graphblas/ops.hpp" "graphblas/phase.hpp"
+	"graphblas/pinnedvector.hpp" "graphblas/properties.hpp" "graphblas/rc.hpp"
+	"graphblas/semiring.hpp" "graphblas/spmd.hpp" "graphblas/tags.hpp"
+	"graphblas/type_traits.hpp" "graphblas/utils.hpp" "graphblas/vector.hpp"
+	"graphblas/synchronizedNonzeroIterator.hpp" "graphblas/nonzeroStorage.hpp"
+	"graphblas/selection_ops.hpp"
 )
 
 set( GRB_INCLUDE_INSTALL_DIR "${INCLUDE_INSTALL_DIR}/graphblas")

include/graphblas/base/blas3.hpp

@@ -511,7 +511,7 @@ namespace grb {
 	 *                        not match. All input data containers are left
 	 *                        untouched if this exit code is returned; it will be
 	 *                        be as though this call was never made.
-	 * @return #grb::FAILED   If \a phase is #grb::EXECUTE, indicates that the
+	 * @return #grb::ILLEGAL  If \a phase is #grb::EXECUTE, indicates that the
 	 *                        capacity of \a B was insufficient. The output
 	 *                        matrix \a B is cleared, and the call to this function
 	 *                        has no further effects.

include/graphblas/base/collectives.hpp

@@ -120,7 +120,7 @@ namespace grb {
 			>
 			static RC allreduce( IOType &inout, const Operator op = Operator() ) {
 				(void) inout;
-				(void)op;
+				(void) op;
 				return PANIC;
 			}
 

include/graphblas/base/final.hpp

@@ -0,0 +1,126 @@
+
+/*
+ *   Copyright 2021 Huawei Technologies Co., Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * @file
+ *
+ * Describes raw functionalities that a so-called <em>final backend</em> should
+ * provide.
+ *
+ * A final backend is one that other backends rely on for implementing core
+ * computations.
+ *
+ * @author A. N. Yzelman
+ * @date 13th of February, 2024
+ */
+
+#ifndef _H_GRB_BASE_FINAL
+#define _H_GRB_BASE_FINAL
+
+#include <string>
+
+#include <graphblas/rc.hpp>
+#include <graphblas/ops.hpp>
+#include <graphblas/backends.hpp>
+#include <graphblas/descriptors.hpp>
+
+
+namespace grb::internal {
+
+	/**
+	 * This class gathers raw functionalities that non-final backends cannot
+	 * implement directly because it is unaware whether final computations should
+	 * occur in parallel or not, while, if it should execute in parallel, it is
+	 * unaware which parallelisation scheme it should employ.
+	 *
+	 * The base implementation defines all functions every final backend should
+	 * implement and provides a sequential implementation for each such function.
+	 * Therefore, only parallel final backends should override this class.
+	 */
+	template< grb::Backend backend >
+	class FinalBackend {
+
+		public:
+
+			/**
+			 * Provides a basic memory copy.
+			 *
+			 * @param[out] out The area where to write to.
+			 * @param[in]  in  The memory area to copy from.
+			 * @param[in] size How many bytes should be copied.
+			 *
+			 * The input and output memory areas are not allowed to overlap.
+			 */
+			static void memcpy(
+				void * const __restrict__ out,
+				const void * const __restrict__ in,
+				const size_t size
+			) {
+				(void) std::memcpy( out, in, size );
+			}
+
+			/**
+			 * Folds (reduces) every column of a matrix into a vector.
+			 *
+			 * @tparam descr  The descriptor to be taken into account.
+			 * @tparam IOType The type of vector \em and matrix elements.
+			 * @tparam OP     The operator used for reduction.
+			 *
+			 * @param[in,out] out The output vector.
+			 *
+			 * Pre-existing values in \a out are reduced into.
+			 *
+			 * @param[in] matrix The matrix which should be column-wise reduced into
+			 *                   \a out.
+			 * @param[in] cols   The colums of \a matrix.
+			 * @param[in] rows   The rows of \a matrix.
+			 * @param[in] skip   Which column of \a matrix to skip.
+			 *
+			 * Taking \a skip higher or equal to \a cols will mean no column is
+			 * skipped.
+			 *
+			 * @param[in] op The operator by which to reduce.
+			 */
+			template< grb::Descriptor descr, typename IOType, typename OP >
+			static void foldMatrixToVector(
+				IOType * const __restrict__ out,
+				const IOType * const __restrict__ matrix,
+				const size_t cols, const size_t rows,
+				const size_t skip,
+				const OP &op
+			) {
+				(void) op;
+				if( skip >= cols ) {
+					for( size_t j = 0; j < cols; ++j ) {
+						OP::eWiseFoldlAA( out, matrix + j * rows, rows );
+					}
+				} else {
+					for( size_t j = 0; j < skip; ++j ) {
+						OP::eWiseFoldlAA( out, matrix + j * rows, rows );
+					}
+					for( size_t j = skip + 1; j < cols; ++j ) {
+						OP::eWiseFoldlAA( out, matrix + j * rows, rows );
+					}
+				}
+			}
+
+	};
+
+}
+
+#endif // end ``_H_GRB_BASE_FINAL´´
+

include/graphblas/base/internalops.hpp

@@ -2836,6 +2836,7 @@ namespace grb {
 					/** The output domain. */
 					typedef typename OP::result_type D3;
 
+
 				public:
 
 					/** @return Whether this operator is mathematically associative. */
@@ -2868,7 +2869,10 @@ namespace grb {
 					 * @param[out] z The output element.
 					 */
 					template< typename InputType1, typename InputType2, typename OutputType >
-					static void apply( const InputType1 & x, const InputType2 & y, OutputType & z ) {
+					static void apply(
+						const InputType1 &x, const InputType2 &y,
+						OutputType &z
+					) {
 						const D1 a = static_cast< D1 >( x );
 						const D2 b = static_cast< D2 >( y );
 						D3 temp;
@@ -2881,7 +2885,7 @@ namespace grb {
 					 * casting is required. This version will be automatically caled whenever
 					 * possible.
 					 */
-					static void apply( const D1 & x, const D2 & y, D3 & out ) {
+					static void apply( const D1 &x, const D2 &y, D3 &out ) {
 						OP::apply( &x, &y, &out );
 					}
 
@@ -2946,7 +2950,7 @@ namespace grb {
 					 * @param[in,out] y The value \a x is to be applied against.
 					 */
 					template< typename InputType, typename IOType >
-					static void foldr( const InputType & x, IOType & y ) {
+					static void foldr( const InputType &x, IOType &y ) {
 						typedef typename OperatorBase< OP >::D2 D2;
 						const D2 cache = static_cast< D2 >( y );
 						OperatorBase< OP >::apply( x, cache, y );
@@ -3110,6 +3114,7 @@ namespace grb {
 					typedef typename OperatorBase< OP >::D3 D3;
 					static constexpr size_t blocksize = OperatorBase< OP >::blocksize;
 
+
 				public:
 
 					/**
@@ -3127,7 +3132,7 @@ namespace grb {
 					 * @param[in]     x The value that is to be applied to \a y.
 					 * @param[in,out] y The value \a x is to be applied against.
 					 */
-					static void foldr( const D1 & x, D3 & y ) {
+					static void foldr( const D1 &x, D3 &y ) {
 						OP::foldr( &x, &y );
 					}
 

include/graphblas/base/io.hpp

@@ -1305,6 +1305,14 @@ namespace grb {
 	 *                  the default is #grb::EXECUTE.
 	 *
 	 * @return #grb::SUCCESS   Upon successful execution of this operation.
+	 * @return #grb::OUTOFMEM  If the capacity of \a x could not be resized to
+	 *                         store a new value at coordinate \a i while \a phase
+	 *                         is #grb::RESIZE.
+	 * @return #grb::ILLEGAL   If the capacity of \a x is insufficient to store a
+	 *                         new value at coordinate \a i while \a phase is
+	 *                         #grb::EXECUTE.
+	 * @return #grb::ILLEGAL   If \a x on input is not dense while the
+	 *                         #grb::descriptors::dense descriptor was given.
 	 * @return #grb::MISMATCH  If \a i is greater or equal than the dimension of
 	 *                         \a x.
 	 *

include/graphblas/base/spmd.hpp

@@ -88,7 +88,9 @@ namespace grb {
 			 *       initiating messages.
 			 * \endinternal
 			 */
-			static enum RC sync( const size_t msgs_in = 0, const size_t msgs_out = 0 ) noexcept {
+			static enum RC sync(
+				const size_t msgs_in = 0, const size_t msgs_out = 0
+			) noexcept {
 				(void) msgs_in;
 				(void) msgs_out;
 				return PANIC;