Extend test suite to run more tests also in pre-commit

.github/workflows/nestbuildmatrix.yml

@@ -10,7 +10,7 @@ jobs:
   clang-format:
     runs-on: "ubuntu-22.04"
     env:
-      CLANG_REQUIRE_VERSION: 17.0.4
+      CLANG_REQUIRE_VERSION: 21.1.7
       CLANG_FORMAT_FILE: ".clang-format"
     steps:
       - name: Harden Runner
@@ -62,11 +62,11 @@ jobs:
       - name: "Set up Python 3.x"
         uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
         with:
-          python-version: 3.9
+          python-version: "3.10"
 
       - name: "Install dependencies"
         run: |
-          pip install pre-commit
+          pip install pre-commit pylint
 
       - name: "Run pre-commit checks..."
         run: |

.pre-commit-config.yaml

@@ -1,6 +1,8 @@
+fail_fast: true
+
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v5.0.0
+    rev: v6.0.0
     hooks:
       - id: end-of-file-fixer
       - id: trailing-whitespace
@@ -10,43 +12,95 @@ repos:
       - id: fix-byte-order-marker
 
   - repo: https://github.com/gitleaks/gitleaks
-    rev: v8.16.3
+    rev: v8.10.0
     hooks:
       - id: gitleaks
 
   - repo: https://github.com/pycqa/isort
-    rev: 5.12.0
+    rev: 7.0.0
     hooks:
       - id: isort
         args: ["--profile", "black", "--thirdparty", "nest"]
 
   - repo: https://github.com/psf/black
-    rev: 23.7.0
+    rev: 25.11.0
     hooks:
       - id: black
         language_version: python3
 
+  - repo: https://github.com/PyCQA/flake8
+    rev: 7.3.0
+    hooks:
+      - id: flake8
+
+  - repo: https://github.com/PyCQA/pydocstyle
+    rev: 6.3.0
+    hooks:
+      - id: pydocstyle
+
   - repo: https://github.com/shellcheck-py/shellcheck-py
-    rev: v0.10.0.1
+    rev: v0.11.0.1
     hooks:
       - id: shellcheck
         # explicitly add input files that are not "*.sh"
         args: [
           #
           # note that these differ from arguments in .github/workflows/nestbuildmatrix.yml
           # IF YOU READ THIS: recheck for consistency with `find . -iname "*.sh" -o -iname "*.sh.in"`
-          "bin/nest_vars.sh.in",   # testsuite/do_tests.sh:112 requires this for `. "${PREFIX}/bin/nest_vars.sh"`
-          "testsuite/run_test.sh",
-          "testsuite/do_tests.sh",
-          "testsuite/junit_xml.sh",
-          "examples/run_examples.sh",
-          "examples/list_examples.sh",
+          "./testsuite/junit_xml.sh",
+          "./testsuite/do_tests.sh",
+          "./testsuite/run_test.sh",
+          "./bin/nest_vars.sh.in",   # testsuite/do_tests.sh:112 requires this for `. "${PREFIX}/bin/nest_vars.sh"`
+          "./cmake/CheckFiles/check_return_val.sh",
+          "./examples/run_examples.sh",
+          "./examples/list_examples.sh",
+          "./build_support/version_info.sh",
+          "./build_support/install_csa-libneurosim.sh",
+          "./build_support/install_music.sh",
+          "./build_support/check_forbidden_types.sh",
+          "./build_support/format_all_c_c++_files.sh",
+          "./build_support/install_sionlib.sh",
         ]
 
   - repo: https://github.com/pre-commit/mirrors-clang-format
-    rev: v17.0.4
+    rev: v21.1.7
     hooks:
       - id: clang-format
         types_or: [c++]
 
+  # ref https://pylint.pycqa.org/en/latest/user_guide/installation/pre-commit-integration.html
+  - repo: local
+    hooks:
+      - id: pylint
+        name: pylint
+        entry: pylint
+        language: system
+        require_serial: true
+        types: ["python"]
+        args:
+            - "-rn"  # Only display messages
+            - "-sn"  # Don't display the score
+            - "--rcfile=.pylintrc"  # Link to your config file
+            - "--load-plugins=pylint.extensions.docparams"  # Load an extension
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.19.0
+    hooks:
+    -   id: mypy
+        additional_dependencies:
+          - 'jinja2'
+          - 'python-dateutil'
+          - 'pytest'
+          - 'semver'
+          - 'types-python-dateutil'
+          - 'types-requests'
+          - 'matplotlib-stubs'
+          - 'pandas-stubs'
+        args:
+          - --show-error-codes
+          - --config-file
+          - pyproject.toml
+          - .
+        pass_filenames: false
+
 exclude: 'build/.*|thirdparty/.*'

.pylintrc

@@ -6,7 +6,7 @@
 options = unneeded-not, line-too-long, unnecessary-semicolon, trailing-whitespace, missing-final-newline, bad-indentation, multiple-statements, bare-except
 ignore = CVS .git conda env __pycache__ .pytest_cache .mypy_cache
 
-disable = no-member, redefined-outer-name, invalid-name, consider-using-f-string, wrong-import-order, missing-function-docstring, missing-method-docstring, missing-class-docstring, attribute-defined-outside-init, no-else-return, cell-var-from-loop, import-error, pointless-string-statement, unused-import, redefined-builtin, superfluous-parens, unused-variable, too-many-locals, consider-using-from-import, consider-using-enumerate, no-name-in-module, too-many-arguments, too-many-instance-attributes, import-outside-toplevel, too-few-public-methods, cyclic-import, missing-module-docstring, unidiomatic-typecheck, dangerous-default-value, unused-argument, use-dict-literal, exec-used, no-self-use, too-many-statements, ungrouped-imports, consider-using-sys-exit, too-many-statements, redundant-u-string-prefix, protected-access, consider-using-dict-comprehension, no-else-raise, too-many-nested-blocks, use-a-generator, reimported, undefined-variable, too-many-branches, raise-missing-from, trailing-comma-tuple, unspecified-encoding, consider-using-with, f-string-without-interpolation, broad-except, unnecessary-pass, global-statement, too-many-lines, consider-merging-isinstance, redefined-argument-from-local, global-variable-undefined, use-implicit-booleaness-not-len, inconsistent-return-statements, consider-using-in, inconsistent-return-statements, keyword-arg-before-vararg, consider-using-dict-items, import-self, fixme, c-extension-no-member, too-many-public-methods, consider-iterating-dictionary, consider-using-max-builtin, super-with-arguments, expression-not-assigned, unnecessary-comprehension, no-self-argument, chained-comparison, undefined-loop-variable, empty-docstring, use-maxsplit-arg, pointless-statement, wrong-import-position, redundant-unittest-assert, eval-used, not-callable, invalid-unary-operand-type, consider-using-generator, R0801, unnecessary-dunder-call, logging-fstring-interpolation, consider-using-get, useless-object-inheritance, unrecognized-option, unknown-option-value, useless-option-value
+disable = no-member, redefined-outer-name, invalid-name, consider-using-f-string, wrong-import-order, missing-function-docstring, missing-method-docstring, missing-class-docstring, attribute-defined-outside-init, no-else-return, cell-var-from-loop, import-error, pointless-string-statement, unused-import, redefined-builtin, superfluous-parens, unused-variable, too-many-locals, consider-using-from-import, consider-using-enumerate, no-name-in-module, too-many-arguments, too-many-instance-attributes, too-many-return-statements, import-outside-toplevel, too-few-public-methods, cyclic-import, missing-module-docstring, unidiomatic-typecheck, dangerous-default-value, unused-argument, use-dict-literal, exec-used, no-self-use, too-many-statements, ungrouped-imports, consider-using-sys-exit, too-many-statements, redundant-u-string-prefix, protected-access, consider-using-dict-comprehension, no-else-raise, too-many-nested-blocks, use-a-generator, reimported, undefined-variable, too-many-branches, raise-missing-from, trailing-comma-tuple, unspecified-encoding, consider-using-with, f-string-without-interpolation, broad-except, unnecessary-pass, global-statement, too-many-lines, consider-merging-isinstance, redefined-argument-from-local, global-variable-undefined, use-implicit-booleaness-not-len, inconsistent-return-statements, consider-using-in, inconsistent-return-statements, keyword-arg-before-vararg, consider-using-dict-items, import-self, fixme, c-extension-no-member, too-many-public-methods, consider-iterating-dictionary, consider-using-max-builtin, super-with-arguments, expression-not-assigned, unnecessary-comprehension, no-self-argument, chained-comparison, undefined-loop-variable, empty-docstring, use-maxsplit-arg, pointless-statement, wrong-import-position, redundant-unittest-assert, eval-used, not-callable, invalid-unary-operand-type, consider-using-generator, R0801, unnecessary-dunder-call, logging-fstring-interpolation, consider-using-get, useless-object-inheritance, unrecognized-option, unknown-option-value, useless-option-value, missing-type-doc, too-many-positional-arguments, missing-param-doc, logging-not-lazy, useless-return, duplicate-key, possibly-used-before-assignment, unnecessary-lambda, consider-using-min-builtin, using-constant-test, unexpected-keyword-arg
 
 const-naming-style = snake_case
 method-naming-style = PascalCase

bin/nest-server-mpi

@@ -13,21 +13,17 @@ Options:
 
 """
 
-from docopt import docopt
-from mpi4py import MPI
-
-if __name__ == "__main__":
-    opt = docopt(__doc__)
-
 import logging
 import os
 import time
 
-logger = logging.getLogger(__name__)
-logger.setLevel(os.getenv("NEST_SERVER_MPI_LOGGER_LEVEL", "INFO"))
-
 import nest
 import nest.server
+from docopt import docopt
+from mpi4py import MPI
+
+logger = logging.getLogger(__name__)
+logger.setLevel(os.getenv("NEST_SERVER_MPI_LOGGER_LEVEL", "INFO"))
 
 HOST = os.getenv("NEST_SERVER_HOST", "127.0.0.1")
 PORT = os.getenv("NEST_SERVER_PORT", "52425")
@@ -41,41 +37,47 @@ def log(call_name, msg):
     logger.debug(msg)
 
 
-if rank == 0:
-    logger.info("==> Starting NEST Server Master on rank 0")
-    nest.server.set_mpi_comm(comm)
-    nest.server.run_mpi_app(host=opt.get("--host", HOST), port=opt.get("--port", PORT))
-
-else:
-    logger.info(f"==> Starting NEST Server Worker on rank {rank}")
-    nest.server.set_mpi_comm(comm)
-
-    while True:
-        log("spinwait", "waiting for call bcast")
-        call_name = comm.bcast(None, root=0)
-
-        log(call_name, "received call bcast, waiting for data bcast")
-        data = comm.bcast(None, root=0)
-
-        log(call_name, f"received data bcast, data={data}")
-        args, kwargs = data
-
-        if call_name == "exec":
-            response = nest.server.do_exec(args, kwargs)
-        else:
-            call, args, kwargs = nest.server.nestify(call_name, args, kwargs)
-            log(call_name, f"local call, args={args}, kwargs={kwargs}")
-
-            # The following exception handler is useful if an error
-            # occurs simulataneously on all processes. If only a
-            # subset of processes raises an exception, a deadlock due
-            # to mismatching MPI communication calls is inevitable on
-            # the next call.
-            try:
-                response = call(*args, **kwargs)
-            except Exception:
-                logger.error("Failed to execute call")
-                continue
-
-        log(call_name, f"sending reponse gather, data={response}")
-        comm.gather(nest.serialize_data(response), root=0)
+def main():
+    opt = docopt(__doc__)
+    if rank == 0:
+        logger.info("==> Starting NEST Server Master on rank 0")
+        nest.server.set_mpi_comm(comm)
+        nest.server.run_mpi_app(host=opt.get("--host", HOST), port=opt.get("--port", PORT))
+
+    else:
+        logger.info(f"==> Starting NEST Server Worker on rank {rank}")
+        nest.server.set_mpi_comm(comm)
+
+        while True:
+            log("spinwait", "waiting for call bcast")
+            call_name = comm.bcast(None, root=0)
+
+            log(call_name, "received call bcast, waiting for data bcast")
+            data = comm.bcast(None, root=0)
+
+            log(call_name, f"received data bcast, data={data}")
+            args, kwargs = data
+
+            if call_name == "exec":
+                response = nest.server.do_exec(args, kwargs)
+            else:
+                call, args, kwargs = nest.server.nestify(call_name, args, kwargs)
+                log(call_name, f"local call, args={args}, kwargs={kwargs}")
+
+                # The following exception handler is useful if an error
+                # occurs simulataneously on all processes. If only a
+                # subset of processes raises an exception, a deadlock due
+                # to mismatching MPI communication calls is inevitable on
+                # the next call.
+                try:
+                    response = call(*args, **kwargs)
+                except Exception:
+                    logger.error("Failed to execute call")
+                    continue
+
+            log(call_name, f"sending reponse gather, data={response}")
+            comm.gather(nest.serialize_data(response), root=0)
+
+
+if __name__ == "__main__":
+    main()

build_support/check_copyright_headers.py

@@ -118,6 +118,9 @@ def eprint(*args, **kwargs):
         if any([exclude_file in tested_file for exclude_file in exclude_files]):
             continue
 
+        if os.stat(tested_file).st_size == 0:
+            continue
+
         with open(tested_file, encoding="utf-8") as source_file:
             total_files += 1
             for template_line in template_contents[extension]:

doc/htmldoc/_ext/extract_api_functions.py

@@ -35,9 +35,9 @@
 
 def find_all_variables(file_path):
     """
-    This function gets the names of all functions listed in ``__all__``
-    in each of the PyNEST API files, along with the Kernel Attributes
-    found in ``__init__.py`` of ``pynest/nest/``.
+    Gets the names of all functions listed in ``__all__`` in each of the PyNEST
+    API files, along with the Kernel Attributes found in ``__init__.py`` of
+    ``pynest/nest/``.
     """
     all_variables = None
 

libnestutil/block_vector.h

@@ -323,15 +323,15 @@ class BlockVector
 template < typename value_type_ >
 BlockVector< value_type_ >::BlockVector()
   : blockmap_(
-    std::vector< std::vector< value_type_ > >( 1, std::move( std::vector< value_type_ >( max_block_size ) ) ) )
+      std::vector< std::vector< value_type_ > >( 1, std::move( std::vector< value_type_ >( max_block_size ) ) ) )
   , finish_( begin() )
 {
 }
 
 template < typename value_type_ >
 BlockVector< value_type_ >::BlockVector( size_t n )
   : blockmap_(
-    std::vector< std::vector< value_type_ > >( 1, std::move( std::vector< value_type_ >( max_block_size ) ) ) )
+      std::vector< std::vector< value_type_ > >( 1, std::move( std::vector< value_type_ >( max_block_size ) ) ) )
   , finish_( begin() )
 {
   size_t num_blocks_needed = std::ceil( static_cast< double >( n ) / max_block_size );

libnestutil/sort.h

@@ -49,12 +49,12 @@ template < typename T >
 inline size_t
 median3_( const BlockVector< T >& vec, const size_t i, const size_t j, const size_t k )
 {
-  return ( ( vec[ i ] < vec[ j ] ) ? ( ( vec[ j ] < vec[ k ] ) ? j
-               : ( vec[ i ] < vec[ k ] )                       ? k
-                                                               : i )
-                                   : ( ( vec[ k ] < vec[ j ] )   ? j
-                                       : ( vec[ k ] < vec[ i ] ) ? k
-                                                                 : i ) );
+  return ( ( vec[ i ] < vec[ j ] ) ? ( ( vec[ j ] < vec[ k ] )     ? j
+                                         : ( vec[ i ] < vec[ k ] ) ? k
+                                                                   : i )
+                                   : ( ( vec[ k ] < vec[ j ] )     ? j
+                                         : ( vec[ k ] < vec[ i ] ) ? k
+                                                                   : i ) );
 }
 
 /**

models/aeif_cond_alpha_multisynapse.cpp

@@ -321,8 +321,8 @@ aeif_cond_alpha_multisynapse::State_::get( DictionaryDatum& d ) const
   std::vector< double >* g = new std::vector< double >();
 
   for ( size_t i = 0;
-        i < ( ( y_.size() - State_::NUMBER_OF_FIXED_STATES_ELEMENTS ) / State_::NUM_STATE_ELEMENTS_PER_RECEPTOR );
-        ++i )
+    i < ( ( y_.size() - State_::NUMBER_OF_FIXED_STATES_ELEMENTS ) / State_::NUM_STATE_ELEMENTS_PER_RECEPTOR );
+    ++i )
   {
     dg->push_back( y_[ State_::DG + ( State_::NUM_STATE_ELEMENTS_PER_RECEPTOR * i ) ] );
     g->push_back( y_[ State_::G + ( State_::NUM_STATE_ELEMENTS_PER_RECEPTOR * i ) ] );

models/aeif_cond_beta_multisynapse.cpp

@@ -329,8 +329,8 @@ aeif_cond_beta_multisynapse::State_::get( DictionaryDatum& d ) const
   std::vector< double >* g = new std::vector< double >();
 
   for ( size_t i = 0;
-        i < ( ( y_.size() - State_::NUMBER_OF_FIXED_STATES_ELEMENTS ) / State_::NUM_STATE_ELEMENTS_PER_RECEPTOR );
-        ++i )
+    i < ( ( y_.size() - State_::NUMBER_OF_FIXED_STATES_ELEMENTS ) / State_::NUM_STATE_ELEMENTS_PER_RECEPTOR );
+    ++i )
   {
     dg->push_back( y_[ State_::DG + ( State_::NUM_STATE_ELEMENTS_PER_RECEPTOR * i ) ] );
     g->push_back( y_[ State_::G + ( State_::NUM_STATE_ELEMENTS_PER_RECEPTOR * i ) ] );

models/cm_tree.cpp

@@ -290,7 +290,7 @@ void
 nest::CompTree::set_parents()
 {
   for ( auto compartment_idx_it = compartment_indices_.begin(); compartment_idx_it != compartment_indices_.end();
-        ++compartment_idx_it )
+    ++compartment_idx_it )
   {
     Compartment* comp_ptr = get_compartment( *compartment_idx_it );
     // will be nullptr if root
@@ -309,7 +309,7 @@ nest::CompTree::set_compartments()
   compartments_.clear();
 
   for ( auto compartment_idx_it = compartment_indices_.begin(); compartment_idx_it != compartment_indices_.end();
-        ++compartment_idx_it )
+    ++compartment_idx_it )
   {
     compartments_.push_back( get_compartment( *compartment_idx_it ) );
   }

models/gif_pop_psc_exp.cpp

@@ -592,7 +592,7 @@ nest::gif_pop_psc_exp::update( Time const& origin, const long from, const long t
       const double ompl = ( 1. - P_lambda_ );
       V_.v_[ k ] = ompl * ompl * V_.v_[ k ] + P_lambda_ * V_.m_[ k ];
       V_.m_[ k ] = ompl * V_.m_[ k ]; // line 26 of [1]
-    }                                 // line 27 of [1]
+    } // line 27 of [1]
 
     double P_Lambda_;
     if ( ( Z_ + V_.z_ ) > 0.0 )

models/glif_cond.cpp

@@ -405,9 +405,9 @@ nest::glif_cond::State_::get( DictionaryDatum& d, const Parameters_& p ) const
   std::vector< double >* dg = new std::vector< double >();
   std::vector< double >* g = new std::vector< double >();
 
-  for ( size_t i = 0; i
-        < ( ( y_.size() - State_::NUMBER_OF_FIXED_STATES_ELEMENTS ) / State_::NUMBER_OF_STATES_ELEMENTS_PER_RECEPTOR );
-        ++i )
+  for ( size_t i = 0;
+    i < ( ( y_.size() - State_::NUMBER_OF_FIXED_STATES_ELEMENTS ) / State_::NUMBER_OF_STATES_ELEMENTS_PER_RECEPTOR );
+    ++i )
   {
     dg->push_back( y_[ State_::DG_SYN - State_::NUMBER_OF_RECORDABLES_ELEMENTS
       + ( i * State_::NUMBER_OF_STATES_ELEMENTS_PER_RECEPTOR ) ] );