fixed pytest and updated documentation references

Author: prasad-sawantdesai

benchmarks/technical.py

@@ -19,9 +19,10 @@ def timeraw_import_imas():
 # `asv compare` :(
 """
 def track_imas_versions():
-    equilibrium = imas.equilibrium()
-    equilibrium.ids_properties.homogeneous_time = imas.imasdef.IDS_TIME_MODE_INDEPENDENT
-    dbentry = imas.DBEntry(imas.imasdef.MEMORY_BACKEND, "test", 1, 1)
+    ids_factory = imas.IDSFactory()
+    equilibrium = ids_factory.equilibrium()
+    equilibrium.ids_properties.homogeneous_time = imas.ids_defs.IDS_TIME_MODE_INDEPENDENT
+    dbentry = imas.DBEntry(imas.ids_defs.MEMORY_BACKEND, "test", 1, 1)
     dbentry.create()
     dbentry.put(equilibrium)
     equilibrium = dbentry.get("equilibrium")

ci/run_benchmark.sh

@@ -59,7 +59,7 @@ asv machine --yes
 # Run ASV for the current commit, develop and main
 asv run --skip-existing-successful HEAD^!
 asv run --skip-existing-successful develop^!
-# asv run --skip-existing-successful main^!
+asv run --skip-existing-successful main^!
 
 # Compare results
 if [ `git rev-parse --abbrev-ref HEAD` == develop ]
@@ -75,5 +75,5 @@ asv publish
 # And persistently store them
 cp -rf .asv/{results,html} "$BENCHMARKS_DIR"
 
-deactivate
+
 

ci/run_pytest.sh

@@ -37,8 +37,10 @@ pip freeze
 # Clean artifacts created by pytest
 rm -f junit.xml
 rm -rf htmlcov
-mkdir -p ~/tmp
-export PYTEST_DEBUG_TEMPROOT=~/tmp
-python -m pytest -n=auto --cov=imas --cov-report=term-missing --cov-report=html --junit-xml=junit.xml -x
 
-deactivate
+# setups local directory to not to full /tmp directory with pytest temporary files
+# mkdir -p ~/tmp
+# export PYTEST_DEBUG_TEMPROOT=~/tmp
+python -m pytest -n=auto --cov=imas --cov-report=term-missing --cov-report=html --junit-xml=junit.xml
+
+

docs/source/benchmarking.rst

@@ -1,4 +1,4 @@
-.. _`benchmarking IMAS`:
+.. _`benchmarking IMAS-Python`:
 
 Benchmarking IMAS-Python
 ========================
@@ -19,15 +19,14 @@ Technical benchmarks
     package.
 
 Basic functional benchmarks
-    These are for benchmarking functionality with an equivalent feature in the IMAS
-    Access Layer HLI. In addition to tracking the performance of the IMAS-Python features
-    over time, we can also benchmark the performance against the traditional HLI.
+    These are for benchmarking functionality with an addition to track the performance 
+    of the IMAS-Python features over time.
 
     For example: putting and getting IDSs.
 
 IMAS-Python-specific functional benchmarks
-    These are for benchmarking functionality without an equivalent feature in the IMAS
-    Access Layer HLI. We use these for tracking the IMAS-Python performance over time.
+    These are for benchmarking core functionalities for checking performance. We use these 
+    for tracking the IMAS-Python core features performance over time.
 
     For example: data conversion between DD versions.
 
@@ -62,33 +61,31 @@ contains tabular results. Some examples:
 .. code-block:: text
     :caption: Example output for a test parametrized in ``hli``
 
-    [ 58.33%] ··· core_profiles.Generate.time_create_core_profiles          ok
-    [ 58.33%] ··· ======== ============
-                    hli                
-                  -------- ------------
-                    imas    22.9±0.4μs 
-                   imas    408±8μs   
-                  ======== ============
+    [56.25%] ··· core_profiles.Generate.time_create_core_profiles                                                                                  ok
+    [56.25%] ··· ====== =============
+                hli                
+                ------ -------------
+                imas   2.04±0.01μs 
+                ====== =============
 
-Here we see the benchmark ``core_profiles.Generate.time_create_core_profiles`` was
-repeated for multiple values of ``hli``: once for the ``imas`` HLI, and once for the
-``imas`` HLI.
+
+Here we see the benchmark ``core_profiles.Generate.time_create_core_profiles`` for 
+imas-python ``imas-python``.
 
 Some benchmarks are parametrized in multiple dimensions, as in below example. This
 results in a 2D table of results.
 
 .. code-block:: text
-    :caption: Example output for a test parametrized in ``hli`` and ``backend``
-
-    [ 70.83%] ··· core_profiles.Get.time_get                                ok
-    [ 70.83%] ··· ======== ========== ============ =========
-                  --                    backend             
-                  -------- ---------------------------------
-                    hli        13          14          11   
-                  ======== ========== ============ =========
-                    imas    75.1±1ms   70.2±0.5ms   207±2ms 
-                   imas   241±4ms     229±2ms     364±6ms 
-                  ======== ========== ============ =========
+    :caption: Example output for a test parametrized in ``imas-python`` and ``backend``
+
+    [65.62%] ··· core_profiles.Get.time_get     ok
+    [65.62%] ··· ====== ========= ========== ============ ========= ============
+                --                             backend                         
+                ------ --------------------------------------------------------
+                hli      HDF5    MDSplus      memory      ASCII      netCDF   
+                ====== ========= ========== ============ ========= ============
+                imas   172±3ms   86.7±2ms   68.5±0.8ms   291±3ms   14.2±0.7ms 
+                ====== ========= ========== ============ ========= ============
 
 .. note::
     The backends are listed by their numerical IDS:
@@ -103,7 +100,7 @@ Running benchmarks (advanced)
 -----------------------------
 
 Running benchmarks quickly, as explained in the previous section, is great during
-development and for comparing the performance of IMAS-Python against the imas HLI. However,
+development and for comparing the performance of IMAS-Python. However,
 ``asv`` can also track the performance of benchmarks over various commits of IMAS-Python.
 Unfortunately this is a bit more tricky to set up.
 
@@ -113,22 +110,7 @@ Setup advanced benchmarking
 
 First, some background on how ``asv`` tracks performance: it creates an isolated virtual
 environment (using the ``virtualenv`` package) and installs IMAS-Python for each commit that
-will be benchmarked. However, because the virtual environment is isolated, the ``imas``
-package won't be available. We need to work around it by setting the environment
-variable ``ASV_PYTHONPATH``:
-
-.. code-block:: console
-    :caption: Setting up the ``ASV_PYTHONPATH`` on SDCC
-
-    $ module load IMAS
-    $ export ASV_PYTHONPATH="$PYTHONPATH"
-
-.. caution::
-
-    ``imas`` must not be available on the ``ASV_PYTHONPATH`` to avoid the interfering
-    of two imas modules (one on the ``PYTHONPATH``, and the other installed by ``asv``
-    in the virtual environment).
-
+will be benchmarked. 
 
 Deciding which commits to benchmark
 '''''''''''''''''''''''''''''''''''
@@ -184,19 +166,14 @@ Instead, you can submit a benchmark job to the compute nodes.
 .. code-block:: bash
     :caption: Benchmark run script (``run_benchmarks.sh``)
 
-    # Load IMAS module
+    # Load IMAS-AL-Core module
     module purge
-    module load IMAS
-    # Verify we can run python and import imas
+    module load IMAS-AL-Core
+    module load Python
+
+    # Verify we can run python
     echo "Python version:"
     python --version
-    echo "Import imas:"
-    python -c 'import imas; print(imas)'
-
-    # Set the ASV_PYTHONPATH so we can `import imas` in the benchmarks
-    export ASV_PYTHONPATH="$PYTHONPATH"
-    echo "ASV_PYTHONPATH=$ASV_PYTHONPATH"
-    echo
 
     # Activate the virtual environment which has asv installed
     . venv_imas/bin/activate

docs/source/changelog.rst

@@ -40,35 +40,35 @@ New features
 - IMAS-Python 1.1 adds support for Identifiers defined by the Data Dictionary. This
   functionality is described in detail in :ref:`Identifiers`.
 - Support for the new
-  :py:const:`~imaspy.ids_defs.FLEXBUFFERS_SERIALIZER_PROTOCOL` that is
+  :py:const:`~imas.ids_defs.FLEXBUFFERS_SERIALIZER_PROTOCOL` that is
   implemented in Access Layer Core 5.3. This is a much faster and more efficient
   serialization format than the
-  :py:const:`~imaspy.ids_defs.ASCII_SERIALIZER_PROTOCOL`. The Flexbuffers
+  :py:const:`~imas.ids_defs.ASCII_SERIALIZER_PROTOCOL`. The Flexbuffers
   serializer protocol requires ``imas_core`` version 5.3 or newer. It is the
   default serializer format when it is available. This features is not available
-  when the variable :py:const:`~imaspy.ids_defs.FLEXBUFFERS_SERIALIZER_PROTOCOL`
+  when the variable :py:const:`~imas.ids_defs.FLEXBUFFERS_SERIALIZER_PROTOCOL`
   is set to ``None``.
 - Preview feature: :ref:`IMAS netCDF files`. Store IDSs in a self-describing
   netCDF file, which can be used for sharing and/or archiving data.
 
   This feature is in `preview` status, meaning that it may change in upcoming
   minor releases of IMAS-Python.
 
-- Additional utility functions in :py:mod:`imaspy.util`:
+- Additional utility functions in :py:mod:`imas.util`:
 
-  - :py:func:`imaspy.util.tree_iter` can be used to iterate over all nodes inside
+  - :py:func:`imas.util.tree_iter` can be used to iterate over all nodes inside
     an IDS.
-  - :py:func:`imaspy.util.get_parent` can be used to get the parent element of
+  - :py:func:`imas.util.get_parent` can be used to get the parent element of
     an IDS node.
-  - :py:func:`imaspy.util.get_time_mode` is a convenience function to get the
+  - :py:func:`imas.util.get_time_mode` is a convenience function to get the
     ``ids_properties/homogeneous_time`` value for any node in the IDS.
-  - :py:func:`imaspy.util.get_toplevel` returns the IDS Toplevel element for any
+  - :py:func:`imas.util.get_toplevel` returns the IDS Toplevel element for any
     node in the IDS.
-  - :py:func:`imaspy.util.is_lazy_loaded` will indicate whether an IDS is lazy
+  - :py:func:`imas.util.is_lazy_loaded` will indicate whether an IDS is lazy
     loaded.
-  - :py:func:`imaspy.util.get_full_path` returns the full path (including Array
+  - :py:func:`imas.util.get_full_path` returns the full path (including Array
     of Structure indices) of a node.
-  - :py:func:`imaspy.util.get_data_dictionary_version` returns the Data
+  - :py:func:`imas.util.get_data_dictionary_version` returns the Data
     Dictionary version for which an IDS was created.
 
 - Add support for IMAS Access Layer Core 5.2 and later. IMAS-Python can now be used
@@ -84,18 +84,18 @@ New features
 
     pip install 'imas[imas-core] @ git+ssh://git@github.com/iterorganization/imas-core.git'
 
-- A diff tool for IDSs: :py:func:`imaspy.util.idsdiff`.
+- A diff tool for IDSs: :py:func:`imas.util.idsdiff`.
 - Implement ``==`` equality checking for IDS Structures and Arrays of Structures
   (`IMAS-5120 <https://jira.iter.org/browse/IMAS-5120>`__).
 - Add option to ignore unknown Data Dictionary versions of data stored in the
   backend.
 
-  During a :py:meth:`~imaspy.db_entry.DBEntry.get` or
-  :py:meth:`~imaspy.db_entry.DBEntry.get_slice`, IMAS-Python first reads the version
+  During a :py:meth:`~imas.db_entry.DBEntry.get` or
+  :py:meth:`~imas.db_entry.DBEntry.get_slice`, IMAS-Python first reads the version
   of the Data Dictionary that was used to store the IDS. When this version is
   not known to IMAS-Python, an error is raised. This error can now be ignored by
   setting the parameter
-  :py:param:`~imaspy.db_entry.DBEntry.get.ignore_unknown_dd_version` to
+  :py:param:`~imas.db_entry.DBEntry.get.ignore_unknown_dd_version` to
   ``True``, and IMAS-Python will do its best to load the data anyway.
 
 - A new command line tool exists for analyzing which Data Dictionary fields are
@@ -128,18 +128,18 @@ Breaking changes
 Bug fixes
 '''''''''
 
-- Fixed a bug in :py:func:`imaspy.util.inspect` when inspecting lazy loaded IDSs.
+- Fixed a bug in :py:func:`imas.util.inspect` when inspecting lazy loaded IDSs.
 - Fixed a bug when converting the ``neutron_diagnostics`` IDS to/from Data
   Dictionary version ``3.41.0``.
 - Fixed a bug that allowed setting arbitrary attributes on IDS structures. It is
   only allowed to use attributes defined by the Data Dictionary.
-- Fixed a bug with :py:func:`~imaspy.ids_toplevel.IDSToplevel.serialize` when
+- Fixed a bug with :py:func:`~imas.ids_toplevel.IDSToplevel.serialize` when
   the IDS is in a non-default Data Dictionary version.
 - Fixed a bug when assigning ``nan`` to a FLT_0D, which would lead to a
   confusing and incorrect log message in IMAS-Python 1.0.
 - Fixed incorrect oldest supported DD version. Previously IMAS-Python indicated that
   DD ``3.21.1`` was supported, however ``3.22.0`` is the oldest Data Dictionary
-  tested (and provided) with IMAS-Python. :py:attr:`imaspy.OLDEST_SUPPORTED_VERSION`
+  tested (and provided) with IMAS-Python. :py:attr:`imas.OLDEST_SUPPORTED_VERSION`
   has been updated to reflect this.
 - Fixed a bug when using numpy functions, such as
   :external:py:func:`numpy.isclose` on scalar numbers. Previously an error was
@@ -157,16 +157,16 @@ Bug fixes
 Improved performance
 ''''''''''''''''''''
 
-- Improved performance of :py:meth:`~imaspy.ids_toplevel.IDSToplevel.validate`.
+- Improved performance of :py:meth:`~imas.ids_toplevel.IDSToplevel.validate`.
 - Improved creation of IMAS-Python IDS objects. This made filling IDSs and loading
-  them with :py:meth:`~imaspy.db_entry.DBEntry.get` /
-  :py:meth:`~imaspy.db_entry.DBEntry.get_slice` 10-20% faster.
+  them with :py:meth:`~imas.db_entry.DBEntry.get` /
+  :py:meth:`~imas.db_entry.DBEntry.get_slice` 10-20% faster.
 - Improved the performance of lazy loading. This is most noticeable with the
   ``HDF5`` backend, which is now up to 40x faster than with IMAS-Python 1.0.
-- Improved the performance of :py:meth:`~imaspy.db_entry.DBEntry.get` /
-  :py:meth:`~imaspy.db_entry.DBEntry.get_slice` /
-  :py:meth:`~imaspy.db_entry.DBEntry.put` /
-  :py:meth:`~imaspy.db_entry.DBEntry.put_slice` for IDSs with many nested arrays
+- Improved the performance of :py:meth:`~imas.db_entry.DBEntry.get` /
+  :py:meth:`~imas.db_entry.DBEntry.get_slice` /
+  :py:meth:`~imas.db_entry.DBEntry.put` /
+  :py:meth:`~imas.db_entry.DBEntry.put_slice` for IDSs with many nested arrays
   of structures. This performance improvement is most noticeable for IDSs with
   filled GGD grids and data structures (up to 25% faster).
 
@@ -195,7 +195,7 @@ recent DD versions. Further details on IDS conversion can be found in
 - Automatically convert data between 0D and 1D when possible (`IMAS-5170
   <https://jira.iter.org/browse/IMAS-5170>`__).
   The following type changes are now automatically supported by
-  :py:func:`imaspy.convert_ids <imaspy.ids_convert.convert_ids>`:
+  :py:func:`imas.convert_ids <imas.ids_convert.convert_ids>`:
 
   - INT_0D to INT_1D
   - FLT_0D to FLT_1D

docs/source/ci_config.rst

@@ -49,7 +49,7 @@ Test with AL<version>
     5.  In the "Script" Task, update the module(s) in the Argument field
 
 Benchmark
-    This job runs the :ref:`ASV benchmarks <benchmarking IMAS>` on the CI server. It
+    This job runs the :ref:`ASV benchmarks <benchmarking IMAS-Python>` on the CI server. It
     is configured such that it can only run on a single CI agent
     (`io-ls-bamboowk6.iter.org`). There are two reasons for this:
 

imas/dd_helpers.py

@@ -134,7 +134,6 @@ def _run_xsl_transformation(
             stylesheet_file=str(xsl_file),
             output_file=str(output_file),
         )
-        logger.info("Transformation complete: %s -> %s", xsd_file, output_file)
 
 
 def build_data_dictionary(repo, tag: str, rebuild=False) -> None:

imas/test/test_dd_helpers.py

@@ -8,6 +8,7 @@
 
 _idsdef_unzipped_relpath = Path("idsdef_unzipped")
 
+
 @pytest.mark.skip(reason="skipping IDSDef.zip generation")
 def test_prepare_data_dictionaries():
     prepare_data_dictionaries()
@@ -50,7 +51,3 @@ def test_prepare_data_dictionaries():
 
     if _build_dir.exists():
         shutil.rmtree(_idsdef_unzipped_relpath)
-
-
-if __name__ == "__main__":
-    pytest.main()