diff --git a/.gitignore b/.gitignore index 81ff18108b..35a8d5ab87 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,7 @@ docs/pyvenv.cfg # Visual Studio .vs/ CMakeSettings.json + +# Python wheels stuff + +*.egg-info/ diff --git a/CMakeLists.txt b/CMakeLists.txt index a26c9b34b6..e17f04f949 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -326,6 +326,8 @@ if(ADIOS2_BUILD_EXAMPLES) add_subdirectory(examples) endif() +add_subdirectory(python) + #------------------------------------------------------------------------------# # Testing #------------------------------------------------------------------------------# diff --git a/bindings/Python/CMakeLists.txt b/bindings/Python/CMakeLists.txt index 5c6121e8f6..423a805995 100644 --- a/bindings/Python/CMakeLists.txt +++ b/bindings/Python/CMakeLists.txt @@ -10,7 +10,7 @@ Python_add_library(adios2_py MODULE py11File.cpp py11File.tcc py11glue.cpp ) -target_compile_definitions(adios2_py PRIVATE "ADIOS2_PYTHON_MODULE_NAME=adios2${ADIOS2_LIBRARY_SUFFIX}") +target_compile_definitions(adios2_py PRIVATE "ADIOS2_PYTHON_MODULE_NAME=adios2_bindings${ADIOS2_LIBRARY_SUFFIX}") if(ADIOS2_HAVE_MPI) target_sources(adios2_py PRIVATE py11ADIOSMPI.cpp @@ -29,25 +29,25 @@ target_link_libraries(adios2_py PRIVATE ${maybe_adios2_cxx11_mpi} adios2_cxx11 ${maybe_adios2_core_mpi} adios2_core adios2::thirdparty::pybind11 - ${maybe_mpi4py} Python::NumPy + ${maybe_mpi4py} ) configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/__init__.py.in - ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/__init__.py + ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings/__init__.py @ONLY ) set_target_properties(adios2_py PROPERTIES CXX_VISIBILITY_PRESET hidden - OUTPUT_NAME adios2${ADIOS2_LIBRARY_SUFFIX} - ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2 - LIBRARY_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2 - RUNTIME_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2 - PDB_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2 - COMPILE_PDB_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2 + OUTPUT_NAME adios2_bindings${ADIOS2_LIBRARY_SUFFIX} + ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings + LIBRARY_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings + PDB_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings + COMPILE_PDB_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings ) -string(REGEX REPLACE "[^/]+" ".." relative_base "${CMAKE_INSTALL_PYTHONDIR}/adios2") +string(REGEX REPLACE "[^/]+" ".." relative_base "${CMAKE_INSTALL_PYTHONDIR}/adios2/bindings") if(CMAKE_SYSTEM_NAME MATCHES "Linux") set_target_properties(adios2_py PROPERTIES INSTALL_RPATH "$ORIGIN/${relative_base}/${CMAKE_INSTALL_LIBDIR}" @@ -55,10 +55,10 @@ if(CMAKE_SYSTEM_NAME MATCHES "Linux") endif() install(TARGETS adios2_py - DESTINATION ${CMAKE_INSTALL_PYTHONDIR}/adios2 + DESTINATION ${CMAKE_INSTALL_PYTHONDIR}/adios2/bindings COMPONENT adios2_python-python ) -install(FILES ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/__init__.py - DESTINATION ${CMAKE_INSTALL_PYTHONDIR}/adios2 +install(FILES ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings/__init__.py + DESTINATION ${CMAKE_INSTALL_PYTHONDIR}/adios2/bindings COMPONENT adios2_python-python ) diff --git a/bindings/Python/__init__.py.in b/bindings/Python/__init__.py.in index 01932e43ce..fb6c0ad344 100644 --- a/bindings/Python/__init__.py.in +++ b/bindings/Python/__init__.py.in @@ -1,3 +1,3 @@ -from .adios2@ADIOS2_LIBRARY_SUFFIX@ import * +from .adios2_bindings@ADIOS2_LIBRARY_SUFFIX@ import * __version__ = "@ADIOS2_VERSION@" diff --git a/bindings/Python/py11glue.cpp b/bindings/Python/py11glue.cpp index 128c52d941..7c81aca1bb 100644 --- a/bindings/Python/py11glue.cpp +++ b/bindings/Python/py11glue.cpp @@ -360,6 +360,24 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m) .def("GetResult", &adios2::py11::Query::GetResult) .def("GetBlockIDs", &adios2::py11::Query::GetBlockIDs); + pybind11::class_(m, "Operator") + // Python 2 + .def("__nonzero__", + [](const adios2::py11::Operator &op) { + const bool opBool = op ? true : false; + return opBool; + }) + // Python 3 + .def("__bool__", + [](const adios2::py11::Operator &op) { + const bool opBool = op ? true : false; + return opBool; + }) + .def("Type", &adios2::py11::Operator::Type) + .def("SetParameter", &adios2::py11::Operator::SetParameter) + .def("Parameters", &adios2::py11::Operator::Parameters); + + pybind11::class_(m, "Variable") // Python 2 .def("__nonzero__", @@ -482,23 +500,6 @@ PYBIND11_MODULE(ADIOS2_PYTHON_MODULE_NAME, m) .def("BlocksInfo", &adios2::py11::Engine::BlocksInfo); - pybind11::class_(m, "Operator") - // Python 2 - .def("__nonzero__", - [](const adios2::py11::Operator &op) { - const bool opBool = op ? true : false; - return opBool; - }) - // Python 3 - .def("__bool__", - [](const adios2::py11::Operator &op) { - const bool opBool = op ? true : false; - return opBool; - }) - .def("Type", &adios2::py11::Operator::Type) - .def("SetParameter", &adios2::py11::Operator::SetParameter) - .def("Parameters", &adios2::py11::Operator::Parameters); - pybind11::class_(m, "File") .def("__repr__", [](const adios2::py11::File &stream) { diff --git a/cmake/DetectOptions.cmake b/cmake/DetectOptions.cmake index efecb57624..05b3d25f56 100644 --- a/cmake/DetectOptions.cmake +++ b/cmake/DetectOptions.cmake @@ -396,12 +396,12 @@ if(NOT SHARED_LIBS_SUPPORTED) endif() if(ADIOS2_USE_Python STREQUAL AUTO) - find_package(Python 3 COMPONENTS Interpreter Development NumPy) + find_package(Python 3 COMPONENTS Interpreter Development) if(Python_FOUND AND ADIOS2_HAVE_MPI) find_package(PythonModule COMPONENTS mpi4py mpi4py/mpi4py.h) endif() elseif(ADIOS2_USE_Python) - find_package(Python 3 REQUIRED COMPONENTS Interpreter Development NumPy) + find_package(Python 3 REQUIRED COMPONENTS Interpreter Development) if(ADIOS2_HAVE_MPI) find_package(PythonModule REQUIRED COMPONENTS mpi4py mpi4py/mpi4py.h) endif() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..dd56276b7a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,43 @@ +[project] +name="adios2" +version="0.0.1" +description="The Adaptable IO System" +authors = [ + {name = "Vicente Adolfo Bolea Sanchez", email = "vicente.bolea@kitware.com"}, +] + +keywords = [ + "Python", + "Web", + "Application", + "Framework", +] +classifiers = [ + "License :: Other/Proprietary License", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3 :: Only", + "Topic :: Software Development :: Libraries :: Application Frameworks", + "Topic :: Software Development :: Libraries :: Python Modules", +] + +dependencies = [ + "numpy", +] + +[project.optional-dependencies] +dev = [ + "pip>=21.3", + "pytest", + "setuptools", + 'black', +] + +[tool.setuptools] +packages = [ "adios2"] + + +[tool.black] +line-length = 99 +target-version = ['py38', 'py39', 'py310'] +include = 'python/adios2/.*.py|testing/adios2/python/.*.py' diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt new file mode 100644 index 0000000000..2c555163fd --- /dev/null +++ b/python/CMakeLists.txt @@ -0,0 +1,13 @@ +#------------------------------------------------------------------------------# +# Distributed under the OSI-approved Apache License, Version 2.0. See +# accompanying file Copyright.txt for details. +#------------------------------------------------------------------------------# + +add_custom_target(python_api ALL COMMAND ${CMAKE_COMMAND} -E copy_directory + ${CMAKE_CURRENT_SOURCE_DIR}/adios2 + ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2) + +install(DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/ + DESTINATION ${CMAKE_INSTALL_PYTHONDIR}/adios2/ + COMPONENT adios2_python-python +) diff --git a/python/adios2/__init__.py b/python/adios2/__init__.py new file mode 100644 index 0000000000..46cd5b16bf --- /dev/null +++ b/python/adios2/__init__.py @@ -0,0 +1,4 @@ +#!/usr/bin/python3 + +import adios2.bindings +import adios2.core diff --git a/python/adios2/adios.py b/python/adios2/adios.py new file mode 100644 index 0000000000..c96aef7b2a --- /dev/null +++ b/python/adios2/adios.py @@ -0,0 +1,42 @@ +from adios2.file import File +from adios2.io import IO +from adios2.operator import Operator + +import adios2.bindings as bindings + + +def open(name, mode, engine_type="BPFile", config_file=None): + """File object open""" + return File(name, mode, engine_type) + + +class ADIOS: + def __init__(self): + self._impl = bindings.ADIOS() + self._operators = {} + self._ios = {} + + def DeclareIO(self, name): + self._ios[name] = IO(self._impl, name) + return self._ios[name] + + def AtIO(self, name): + return self._ios[name] + + def RemoveIO(self, name): + del self._ios[name] + self._impl.RemoveIO(name) + + def RemoveAllIOs(self): + self._ios = {} + self._impl.RemoveAllIOs() + + def DefineOperator(self, name, type, parameters={}): + self._operators[name] = Operator(self._impl, name, type, parameters) + return self._operators[name] + + def InquireOperator(self, name): + return self._operators[name] + + def FlushAll(self): + self._impl.FlushAll() diff --git a/python/adios2/attribute.py b/python/adios2/attribute.py new file mode 100644 index 0000000000..0a1277822a --- /dev/null +++ b/python/adios2/attribute.py @@ -0,0 +1,20 @@ +import adios2.bindings + + +class Attribute: + def __init__(self, io, name, *args, **kwargs): + self._impl = io.DefineAttribute(name, *args, **kwargs) + + def __eq__(self, other): + if isinstance(other, Attribute): + return self.Name() == other.Name() + return False + + def Name(self): + return self._impl.Name() + + def Data(self): + return self._impl.Data() + + def DataString(self): + return self._impl.DataString() diff --git a/python/adios2/engine.py b/python/adios2/engine.py new file mode 100644 index 0000000000..a5e6e45e5d --- /dev/null +++ b/python/adios2/engine.py @@ -0,0 +1,65 @@ +import numpy as np +import adios2.bindings as bindings + +from adios2.attribute import Attribute +from adios2.variable import Variable + + +class Engine: + def __init__(self, io, name, mode): + self._impl = io.Open(name, mode) + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.Close() + + def Close(self, transportIndex=-1): + self._impl.Close(transportIndex) + + def Steps(self): + return self._impl.Steps() + + def CurrentStep(self): + return self._impl.CurrentStep() + + def BeginStep(self, *args, **kwargs): + return self._impl.BeginStep(*args, **kwargs) + + def EndStep(self): + self._impl.EndStep() + + def BlocksInfo(self, name, step): + return self._impl.BlocksInfo(name, step) + + def Put(self, variable, content, mode=bindings.Mode.Deferred): + if isinstance(content, np.ndarray): + self._impl.Put(variable._impl, content, mode) + else: + self._impl.Put(variable._impl, content) + + def PerformPuts(self): + self._impl.PerformPuts() + + def PerformDataWrite(self): + self._impl.PerformDataWrite() + + def Get(self, variable, content=None, mode=bindings.Mode.Deferred): + if isinstance(content, np.ndarray): + self._impl.Get(variable._impl, content, mode) + return None + else: + return self._impl.Get(variable._impl, mode) + + def PerformGets(self): + self._impl.PerformGets() + + def LockReaderSelections(self): + self._impl.LockReaderSelections() + + def LockWriterDefinitions(self): + self._impl.LockWriterDefinitions() + + def Flush(self, transportIndex=-1): + self._impl.Flush(transportIndex) diff --git a/python/adios2/file.py b/python/adios2/file.py new file mode 100644 index 0000000000..ca2dfebf20 --- /dev/null +++ b/python/adios2/file.py @@ -0,0 +1,500 @@ +#!/usr/bin/python3 + + +def Params(): + return None + + +def Dims(): + return None + + +class File: + def __init__(self, name, mode, engine_type, conflig_file=None): + pass + + def set_parameter(self, key, value): + """ + Sets a single parameter. Overwrites value if key exists. + + Parameters + key + input parameter key + + value + parameter value + """ + pass + + def set_parameters(self, **kwargs): + """ + Sets parameters using a dictionary. + Removes any previous parameter. + + Parameters + parameters + input key/value parameters + + value + parameter value + """ + pass + + def add_transport(self, transport, parameters=Params()): + """ + + Adds a transport and its parameters to current IO. Must be + supported by current engine type. + + Parameters + type + must be a supported transport type for current engine. + + parameters + acceptable parameters for a particular transport + CAN'T use the keywords "Transport" or "transport" in key + + Returns + transport_index + handler to added transport + """ + pass + + def available_variables(keys): + """ + + Returns a 2-level dictionary with variable information. + Read mode only. + + Parameters + keys + list of variable information keys to be extracted (case insensitive) + keys=['AvailableStepsCount','Type','Max','Min','SingleValue','Shape'] + keys=['Name'] returns only the variable names as 1st-level keys + leave empty to return all possible keys + + Returns + variables dictionary + key + variable name + value + variable information dictionary + """ + pass + + def available_attributes(): + """ + pybind11::return_value_policy::move, R"md( + Returns a 2-level dictionary with attribute information. + Read mode only. + + Returns + attributes dictionary + key + attribute name + value + attribute information dictionary + """ + pass + + def write( + name, + array, + shape=Dims(), + start=Dims(), + count=Dims(), + end_step=False, + ): + """ + writes a self-describing array (numpy) variable + + Parameters + name + variable name + + array + variable data values + + shape + variable global MPI dimensions. + Pass empty numpy array for local variables. + + start + variable offset for current MPI rank. + Pass empty numpy array for local variables. + + count + variable dimension for current MPI rank. + Pass a numpy array for local variables. + + end_step + end current step, begin next step and flush (default = false). + """ + pass + + def write(name, array, shape, start, count, operations, end_step=False): + """ + writes a self-describing array (numpy) variable with operations + e.g. compression: 'zfp', 'mgard', 'sz' + + Parameters + name + variable name + + array + variable data values + + shape + variable global MPI dimensions. + Pass empty numpy array for local variables. + + start + variable offset for current MPI rank. + Pass empty numpy array for local variables. + + count + variable dimension for current MPI rank. + Pass a numpy array for local variables. + + end_step + end current step, begin next step and flush (default = false). + """ + pass + + def write(name, array, local_value=False, end_step=False): + """ + + writes a self-describing single value array (numpy) variable + + Parameters + name + variable name + + array + variable data single value + + local_value + true: local value, false: global value + + end_step + end current step, begin next step and flush + (default = false). + """ + pass + + def write(name, string, local_value=False, end_step=False): + """ + writes a self-describing single value string variable + + Parameters + name + variable name + + string + variable data single value + + local_value + true: local value, false: global value + + end_step + end current step, begin next step and flush + (default = false). + """ + pass + + def write_attribute(name, array, variable_name="", separator="/", end_step=False): + """ + writes a self-describing single value array (numpy) variable + + Parameters + name + attribute name + + array + attribute numpy array data + + variable_name + if attribute is associated with a variable + + separator + concatenation string between variable_name and attribute + e.g. variable_name + separator + name ("var/attr") + Not used if variable_name is empty + + end_step + end current step, begin next step and flush + (default = false). + """ + pass + + def write_attribute(name, string_value, variable_name="", separator="/", end_step=False): + """ + writes a self-describing single value array (numpy) variable + + Parameters + name + attribute name + + string_value + attribute single string + + variable_name + if attribute is associated with a variable + + separator + concatenation string between variable_name and attribute + e.g. variable_name + separator + name ("var/attr") + Not used if variable_name is empty + + end_step + end current step, begin next step and flush + (default = false). + """ + pass + + def write_attribute(name, string_array, variable_name="", separator="/", end_step=False): + """ + writes a self-describing single value array (numpy) variable + + Parameters + name + attribute name + + string_array + attribute string array + + variable_name + if attribute is associated with a variable + + separator + concatenation string between variable_name and attribute + e.g. variable_name + separator + name ("var/attr") + Not used if variable_name is empty + + end_step + end current step, begin next step and flush + (default = false). + """ + pass + + def read_string(name, block_id=0): + """ + Reads string value for current step + (use for streaming mode step by step) + + Parameters + name + string variable name + + block_id + required for local variables + + Returns + + list + data string values. + For global values: returns 1 element + For local values: returns n-block elements + + """ + pass + + def read_string(name, step_start, step_count, block_id=0): + """ + R"md( + Reads string value for a certain step + (random access mode) + + Parameters + name + string variable name + + step_start + variable step start + + step_count + variable number of steps to read from step_start + + block_id + required for local variables + + Returns + string + data string values for a certain step range. + """ + pass + + def read(name, block_id=0): + """ + Reads entire variable for current step + (streaming mode step by step) + + Parameters + name + variable name + + block_id + required for local array variables + + Returns + array + values of variable name for current step. + Single values will have a shape={1} numpy array + """ + pass + + def read(start=Dims(), count=Dims(), block_id=0): + """ + Reads a selection piece in dimension for current step + (streaming mode step by step) + + Parameters + name + variable name + + start + variable local offset selection (defaults to (0, 0, ...) + + count + variable local dimension selection from start + defaults to whole array for GlobalArrays, or selected Block size + for LocalArrays + + block_id + required for local array variables + + Returns + array + values of variable name for current step + empty if exception is thrown + """ + pass + + def read(name, start, count, step_start, step_count, block_id=0): + """ + Random access read allowed to select steps, + only valid with File Engines + + Parameters + name + variable to be read + + start + variable offset dimensions + + count + variable local dimensions from offset + + step_start + variable step start + + step_count + variable number of steps to read from step_start + + block_id + required for local array variables + + Returns + array + resulting array from selection + """ + pass + + def read_attribute(name, variable_name, separator="/"): + """ + Reads a numpy based attribute + + Parameters + name + attribute name + + variable_name + if attribute is associated with a variable + + separator + concatenation string between variable_name and attribute + e.g. variable_name + separator + name (var/attr) + Not used if variable_name is empty + + Returns + array + resulting array attribute data + """ + pass + + def read_attribute_string(name, variable_name="", separator="/"): + """ + Read a string attribute + + Parameters + name + attribute name + + variable_name + if attribute is associated with a variable + + separator + concatenation string between variable_name and attribute + e.g. variable_name + separator + name (var/attr) + Not used if variable_name is empty + + Returns + list + resulting string list attribute data) + """ + pass + + def end_step(): + """ + Write mode: advances to the next step. Convenient when declaring + variable attributes as advancing to the next step is not attached + to any variable. + + Read mode: in streaming mode releases the current step (no effect + in file based engines) + """ + pass + + def close(): + """ + Closes file, thus becoming unreachable. + Not required if using open in a with-as statement. + Required in all other cases per-open to avoid resource leaks. + """ + pass + + def current_step(): + """ + Inspect current step when using for-in loops, read mode only + + Returns + current step + """ + pass + + def steps(): + """ + Inspect available number of steps, for file engines, read mode only + + Returns + steps + """ + pass + + def __repr__(self): + return f"" + + def __enter__(self): + return self + + def __exit__(self): + self.close() + + def __iter__(self): + return self + + def __next__(self): + if not self.GetStep(): + raise stop_iteration() + + return self diff --git a/python/adios2/io.py b/python/adios2/io.py new file mode 100644 index 0000000000..a9336110dd --- /dev/null +++ b/python/adios2/io.py @@ -0,0 +1,103 @@ +import numpy as np +import adios2.bindings as bindings +from adios2.attribute import Attribute +from adios2.variable import Variable +from adios2.engine import Engine + + +class IO: + def __init__(self, adios, name): + self._impl = adios.DeclareIO(name) + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.FlushAll() + + def DefineAttribute( + self, + name, + content=None, + variable_name="", + separator="/", + ): + return Attribute(self._impl, name, content, variable_name, separator) + + def InquireAttribute(self, name): + attr = None + attr_impl = self._impl.InquireAttribute(name) + if attr_impl: + attr = Attribute.__new__(Attribute) + attr._impl = attr_impl + return attr + + def AvailableAttributes(self): + attributes = {} + for name, attr in self._impl.AvailableAttributes: + attribute[name] = Attribute(attr) + return attributes + + def RemoveAttribute(self, name): + self._impl.RemoveAttribute(name) + + def RemoveAllAttributes(self): + self._impl.RemoveAllAttributes() + + def DefineVariable( + self, + name, + content=None, + shape=[0, 0], + start=[0, 0], + count=[0, 0], + isConstantDims=bindings.ConstantDims, + ): + if isinstance(content, np.ndarray): + return Variable(self._impl, name, content, shape, start, count, isConstantDims) + else: + return Variable(self._impl, name) + + def InquireVariable(self, name): + var = None + var_impl = self._impl.InquireVariable(name) + if var_impl: + var = Variable.__new__(Variable) + var._impl = var_impl + return var + + def AvailableVariables(self): + variables = {} + for name, attr in self._impl.AvailableVariables: + variables[name] = Variable(attr) + return variables + + def RemoveVariable(self, name): + self._impl.RemoveVariable(name) + + def RemoveAllVariables(self): + self._impl.RemoveAllVariables() + + def Open(self, name, mode): + return Engine(self._impl, name, mode) + + def SetEngine(self, name): + self._impl.SetEngine(name) + + def EngineType(self): + return self._impl.EngineType() + + def AddTransport(self, type, parameters=dict()): + self._impl.AddTransport(type, parameters) + + def Parameters(self): + return self._impl.Parameters() + + def SetParameter(self, key, value): + self._impl.SetParameter(key, value) + + def SetParameters(self, parameters): + self._impl.SetParameters(parameters) + + def FlushAll(self): + pass diff --git a/python/adios2/operator.py b/python/adios2/operator.py new file mode 100644 index 0000000000..f217904991 --- /dev/null +++ b/python/adios2/operator.py @@ -0,0 +1,12 @@ +import adios2.bindings + + +class Operator: + def __init__(self, adios_instance, name, type, parameters={}): + self._impl = adios_instance.DefineOperator(name, type, parameters) + + def GetParameters(self): + return self._impl.Parameters() + + def SetParameter(self, key, value): + self._impl.SetParameter(key, value) diff --git a/python/adios2/variable.py b/python/adios2/variable.py new file mode 100644 index 0000000000..6565a2c054 --- /dev/null +++ b/python/adios2/variable.py @@ -0,0 +1,59 @@ +import adios2.bindings + + +class Variable: + def __init__(self, io, name, *args, **kwargs): + self._impl = io.DefineVariable(name, *args, **kwargs) + + def __eq__(self, other): + if isinstance(other, Variable): + return self.Name() == other.Name() + return False + + def BlockID(self): + return self._impl.BlockID() + + def Count(self): + return self._impl.Count() + + def SelectionSize(self): + return self._impl.SelectionSize() + + def SetBlockSelection(self, block_id): + self._impl.SetBlockSelection(block_id) + + def SetSelection(self, selection): + self._impl.SetSelection(selection) + + def SetShape(self, shape): + self._impl.SetShape(shape) + + def SetStepSelection(self, step_selection): + self._impl.SetStepSelection(step_selection) + + def Shape(self, step=0): + return self._impl.Shape(step) + + def ShapeID(self): + return self._impl.ShapeID() + + def Sizeof(self): + return self._impl.Sizeof() + + def Start(self): + return self._impl.Start() + + def Steps(self): + return self._impl.Steps() + + def StepsStart(self): + return self._impl.StepsStart() + + def Name(self): + return self._impl.Name() + + def AddOperation(self, op, params={}): + return self._impl.AddOperation(op._impl, params) + + def Operations(self): + return self._impl.Operations() diff --git a/testing/adios2/CMakeLists.txt b/testing/adios2/CMakeLists.txt index dc1ede6bca..6e94ac1102 100644 --- a/testing/adios2/CMakeLists.txt +++ b/testing/adios2/CMakeLists.txt @@ -16,3 +16,7 @@ add_subdirectory(backward_compatibility) if (ADIOS2_HAVE_Derived_Variable) add_subdirectory(derived) endif() + +if (ADIOS2_HAVE_Python) +add_subdirectory(python) +endif() diff --git a/testing/adios2/bindings/python/TestBPWriteReadTypes_nompi.py b/testing/adios2/bindings/python/TestBPWriteReadTypes_nompi.py index 28ca3730bd..b1d1ffe4a2 100644 --- a/testing/adios2/bindings/python/TestBPWriteReadTypes_nompi.py +++ b/testing/adios2/bindings/python/TestBPWriteReadTypes_nompi.py @@ -10,7 +10,7 @@ from adios2NPTypes import SmallTestData -import adios2 +import adios_bindings # Test data diff --git a/testing/adios2/bindings/python/TestBPWriteTypesHighLevelAPI.py b/testing/adios2/bindings/python/TestBPWriteTypesHighLevelAPI.py index d3eb4b58dc..4cb67bd1d7 100644 --- a/testing/adios2/bindings/python/TestBPWriteTypesHighLevelAPI.py +++ b/testing/adios2/bindings/python/TestBPWriteTypesHighLevelAPI.py @@ -12,7 +12,7 @@ from adios2NPTypes import SmallTestData from mpi4py import MPI import numpy as np -import adios2 +import adios comm = MPI.COMM_WORLD rank = comm.Get_rank() diff --git a/testing/adios2/python/CMakeLists.txt b/testing/adios2/python/CMakeLists.txt new file mode 100644 index 0000000000..12e6d40c04 --- /dev/null +++ b/testing/adios2/python/CMakeLists.txt @@ -0,0 +1,11 @@ +#------------------------------------------------------------------------------# +# Distributed under the OSI-approved Apache License, Version 2.0. See +# accompanying file Copyright.txt for details. +#------------------------------------------------------------------------------# + +python_add_test(NAME Api.Python.ADIOS SCRIPT TestADIOS.py) +python_add_test(NAME Api.Python.Engine SCRIPT TestEngine.py) +python_add_test(NAME Api.Python.IO SCRIPT TestIO.py) +python_add_test(NAME Api.Python.Operator SCRIPT TestOperator.py) +python_add_test(NAME Api.Python.Variable SCRIPT TestVariable.py) +python_add_test(NAME Api.Python.Attribute SCRIPT TestAttribute.py) diff --git a/testing/adios2/python/TestADIOS.py b/testing/adios2/python/TestADIOS.py new file mode 100644 index 0000000000..38706dbe55 --- /dev/null +++ b/testing/adios2/python/TestADIOS.py @@ -0,0 +1,61 @@ +from adios2.adios import ADIOS +from adios2 import adios +import unittest + + +class TestADIOS(unittest.TestCase): + def test_open(self): + file = adios.open("io1", "read") + self.assertNotEqual(file, None) + + def test_define_operator(self): + adios = ADIOS() + op = adios.DefineOperator("op", "null") + self.assertNotEqual(op, None) + + def test_inquiry_operator(self): + adios = ADIOS() + op1 = adios.DefineOperator("op1", "null") + op2 = adios.DefineOperator("op2", "null") + op_x = adios.InquireOperator("op2") + self.assertNotEqual(op1, op2) + self.assertEqual(op2, op_x) + self.assertNotEqual(op1, op_x) + with self.assertRaises(KeyError): + adios.InquireOperator("NonExisting") + + def test_declare_io(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + self.assertNotEqual(writer, None) + + def test_at_io(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + reader = adios.DeclareIO("BPReader") + x = adios.AtIO("BPReader") + self.assertNotEqual(writer, reader) + self.assertEqual(reader, x) + self.assertNotEqual(writer, x) + with self.assertRaises(KeyError): + adios.AtIO("NonExisting") + + def test_remove_io(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + adios.RemoveIO("BPWriter") + with self.assertRaises(KeyError): + adios.AtIO("BPWriter") + + def test_remove_all_io(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + reader = adios.DeclareIO("BPReader") + adios.RemoveAllIOs() + with self.assertRaises(KeyError): + adios.AtIO("BPWriter") + adios.AtIO("BPReader") + + +if __name__ == "__main__": + unittest.main() diff --git a/testing/adios2/python/TestAttribute.py b/testing/adios2/python/TestAttribute.py new file mode 100644 index 0000000000..07db0e3f66 --- /dev/null +++ b/testing/adios2/python/TestAttribute.py @@ -0,0 +1,45 @@ +from adios2.adios import ADIOS +from adios2 import adios + +import adios2.bindings as bindings + +import unittest +import numpy as np + + +class TestAttribute(unittest.TestCase): + def test_create_write(self): + adios = ADIOS() + with adios.DeclareIO("BPWriter") as writer: + ts = writer.DefineAttribute("timestamp", "20231122") + self.assertEqual(ts.Name(), "timestamp") + self.assertEqual(ts.DataString(), ["20231122"]) + + def test_create_reader(self): + adios = ADIOS() + with adios.DeclareIO("BPWriter") as writer: + ts = writer.DefineAttribute("timestamp", "20231122") + self.assertEqual(ts.Name(), "timestamp") + self.assertEqual(ts.DataString(), ["20231122"]) + + with adios.DeclareIO("BPReader") as reader: + ts = writer.InquireAttribute("timestamp") + self.assertEqual(ts.Name(), "timestamp") + self.assertEqual(ts.DataString(), ["20231122"]) + + def test_create_write_ndarray(self): + adios = ADIOS() + with adios.DeclareIO("BPWriter") as writer: + arr = np.array([2023, 11, 22]) + ts = writer.DefineAttribute("timestamp", arr) + self.assertEqual(ts.Name(), "timestamp") + self.assertTrue(np.array_equal(ts.Data(), [2023, 11, 22])) + + with adios.DeclareIO("BPReader") as reader: + ts = writer.InquireAttribute("timestamp") + self.assertEqual(ts.Name(), "timestamp") + self.assertTrue(np.array_equal(ts.Data(), [2023, 11, 22])) + + +if __name__ == "__main__": + unittest.main() diff --git a/testing/adios2/python/TestEngine.py b/testing/adios2/python/TestEngine.py new file mode 100644 index 0000000000..1cf5e6fca6 --- /dev/null +++ b/testing/adios2/python/TestEngine.py @@ -0,0 +1,92 @@ +from adios2.adios import ADIOS +from adios2 import adios + +import adios2.bindings as bindings + +import unittest +import numpy as np + + +class TestEngine(unittest.TestCase): + def test_close(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + with writer.Open("pythontestengine.bp", bindings.Mode.Write) as engine: + pass + + def test_put(self): + adios = ADIOS() + with adios.DeclareIO("BPWriter") as writer: + pressure = writer.DefineVariable("pressure") + temps = writer.DefineVariable("temps", np.empty([4], dtype=np.int64)) + with writer.Open("pythontestengine.bp", bindings.Mode.Write) as engine: + engine.Put(pressure, "35PSI") + temps_measures = np.array([35, 40, 30, 45], dtype=np.int64) + engine.Put(temps, temps_measures) + + def test_get(self): + adios = ADIOS() + with adios.DeclareIO("BPWriter") as writer: + pressure = writer.DefineVariable("pressure") + temps = writer.DefineVariable( + name="temps", content=np.empty([4], dtype=np.int64), shape=[1, 4], count=[1, 4] + ) + with writer.Open("pythontestengine.bp", bindings.Mode.Write) as engine: + engine.Put(pressure, "35PSI") + temps_measures = np.array([35, 40, 30, 45], dtype=np.int64) + engine.Put(temps, temps_measures) + + with adios.DeclareIO("BPReader") as reader: + with reader.Open("pythontestengine.bp", bindings.Mode.Read) as engine: + engine.BeginStep() + pressure = reader.InquireVariable("pressure") + temps = reader.InquireVariable("temps") + pressure_reading = engine.Get(pressure) + temps_reading = np.empty([4], dtype=np.int64) + engine.Get(temps, temps_reading) + engine.EndStep() + self.assertEqual(pressure_reading, "35PSI") + self.assertTrue(np.array_equal(temps_reading, np.array([35, 40, 30, 45]))) + + def test_steps(self): + adios = ADIOS() + with adios.DeclareIO("BPWriter") as writer: + pressure = writer.DefineVariable("pressure") + with writer.Open("pythontestengine.bp", bindings.Mode.Write) as engine: + for step in range(0, 10): + engine.BeginStep() + engine.Put(pressure, f"{step}PSI") + engine.EndStep() + + with adios.DeclareIO("BPReader") as reader: + with reader.Open("pythontestengine.bp", bindings.Mode.Read) as engine: + for i in range(0, engine.Steps()): + engine.BeginStep() + pressure = reader.InquireVariable("pressure") + pressure_reading = engine.Get(pressure) + self.assertEqual(pressure_reading, f"{i}PSI") + engine.EndStep() + + def test_blockinfo(self): + adios = ADIOS() + with adios.DeclareIO("BPWriter") as writer: + temps = writer.DefineVariable("temps", np.empty([4], dtype=np.int64)) + with writer.Open("pythontestengine.bp", bindings.Mode.Write) as engine: + temps_measures = np.array([35, 40, 30, 45], dtype=np.int64) + engine.Put(temps, temps_measures) + + with adios.DeclareIO("BPReader") as reader: + with reader.Open("pythontestengine.bp", bindings.Mode.Read) as engine: + engine.BeginStep() + temps = reader.InquireVariable("temps") + info = engine.BlocksInfo("temps", 0) + engine.EndStep() + + self.assertIsNot(info, None) + self.assertTrue(info[0]["Start"], "0,0") + self.assertTrue(info[0]["Count"], "0,0") + self.assertTrue(info[0]["WriterID"], "0") + + +if __name__ == "__main__": + unittest.main() diff --git a/testing/adios2/python/TestIO.py b/testing/adios2/python/TestIO.py new file mode 100644 index 0000000000..0209527708 --- /dev/null +++ b/testing/adios2/python/TestIO.py @@ -0,0 +1,102 @@ +from adios2.adios import ADIOS +import adios2.bindings as bindings + +import unittest + + +class TestIO(unittest.TestCase): + def test_io_empty(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + + def test_io_define_attribute(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + ts = writer.DefineAttribute("timestamp", "20231122") + self.assertIsNot(ts, None) + + def test_io_inquire_attribute(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + ts = writer.DefineAttribute("timestamp", "20231122") + coords = writer.DefineAttribute("coords", "43N74W") + x = writer.InquireAttribute("coords") + self.assertNotEqual(ts, coords) + self.assertNotEqual(ts, x) + self.assertEqual(coords, x) + + def test_available_attribute(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + ts = writer.DefineAttribute("timestamp", "20231122") + writer.InquireAttribute("timestamp") + self.assertIs(writer.InquireAttribute("coords"), None) + + def test_remove_attribute(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + ts = writer.DefineAttribute("timestamp", "20231122") + writer.RemoveAttribute("timestamp") + self.assertIs(writer.InquireAttribute("timestamp"), None) + + def test_remove_all_attribute(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + ts = writer.DefineAttribute("timestamp", "20231122") + coords = writer.DefineAttribute("coords", "43N74W") + writer.RemoveAllAttributes() + self.assertIs(writer.InquireAttribute("timestamp"), None) + self.assertIs(writer.InquireAttribute("coords"), None) + + def test_io_define_variable(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + temp = writer.DefineVariable("temp") + self.assertNotEqual(temp, None) + + def test_io_inquire_variable(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + temp = writer.DefineVariable("temp") + presure = writer.DefineVariable("pressure") + x = writer.InquireVariable("pressure") + self.assertNotEqual(temp, presure) + self.assertNotEqual(temp, x) + self.assertEqual(presure, x) + + def test_available_variable(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + temp = writer.DefineVariable("temp") + writer.InquireVariable("temp") + self.assertIs(writer.InquireAttribute("pressure"), None) + + def test_remove_variable(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + temp = writer.DefineVariable("temp") + writer.RemoveVariable("temp") + self.assertIs(writer.InquireAttribute("temp"), None) + + def test_remove_all_variable(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + temp = writer.DefineVariable("temp") + presure = writer.DefineVariable("pressure") + writer.RemoveAllVariables() + self.assertIs(writer.InquireAttribute("pressure"), None) + self.assertIs(writer.InquireAttribute("temp"), None) + + def test_open_engine(self): + adios = ADIOS() + writer = adios.DeclareIO("BPWriter") + writer.SetEngine("BPFile") + writer.SetParameter("threads", "2") + writer.SetParameters({"AsyncOpen": "On", "MaxOpenFilesAtOnce": "512"}) + writer.AddTransport("File", {"Library": "POSIX"}) + engine = writer.Open("pythontest.bp", bindings.Mode.Write) + self.assertNotEqual(engine, None) + + +if __name__ == "__main__": + unittest.main() diff --git a/testing/adios2/python/TestOperator.py b/testing/adios2/python/TestOperator.py new file mode 100644 index 0000000000..4e64592659 --- /dev/null +++ b/testing/adios2/python/TestOperator.py @@ -0,0 +1,38 @@ +from adios2.adios import ADIOS + +import adios2.bindings as bindings + +import unittest +import numpy as np + + +class TestOperator(unittest.TestCase): + def test_operator_basic(self): + adios = ADIOS() + op1 = adios.DefineOperator("noop", "null") + with adios.DeclareIO("BPWriter") as writer: + temps = writer.DefineVariable("temps", np.empty([4], dtype=np.int64)) + temps.AddOperation(op1) + with writer.Open("pythontestvariable.bp", bindings.Mode.Write) as engine: + temps_measures = np.array([35, 40, 30, 45], dtype=np.int64) + engine.Put(temps, temps_measures) + + op2 = adios.DefineOperator("noop2", "null") + with adios.DeclareIO("BPReader") as reader: + with reader.Open("pythontestvariable.bp", bindings.Mode.Read) as engine: + engine.BeginStep() + temps = reader.InquireVariable("temps") + temps.AddOperation(op2) + engine.EndStep() + + def test_operator_params(self): + adios = ADIOS() + op = adios.DefineOperator("noop", "null") + op.SetParameter("speed", "best") + op.GetParameters() + self.assertTrue("speed" in op.GetParameters()) + self.assertEqual(op.GetParameters()["speed"], "best") + + +if __name__ == "__main__": + unittest.main() diff --git a/testing/adios2/python/TestVariable.py b/testing/adios2/python/TestVariable.py new file mode 100644 index 0000000000..44c1b696df --- /dev/null +++ b/testing/adios2/python/TestVariable.py @@ -0,0 +1,67 @@ +from adios2.adios import ADIOS +from adios2 import adios + +import adios2.bindings as bindings + +import unittest +import numpy as np + + +class TestVariable(unittest.TestCase): + def test_create_write(self): + adios = ADIOS() + with adios.DeclareIO("BPWriter") as writer: + temps = writer.DefineVariable("temps", np.empty([4], dtype=np.int64)) + with writer.Open("pythontestvariable.bp", bindings.Mode.Write) as engine: + temps_measures = np.array([35, 40, 30, 45], dtype=np.int64) + engine.Put(temps, temps_measures) + self.assertEqual(temps.Name(), "temps") + self.assertEqual(temps.BlockID(), 0) + self.assertEqual(temps.Count(), [0, 0]) + self.assertEqual(temps.Shape(), [0, 0]) + self.assertEqual(temps.Sizeof(), 8) + self.assertEqual(temps.Steps(), 1) + self.assertEqual(temps.StepsStart(), 0) + + def test_create_reader(self): + adios = ADIOS() + with adios.DeclareIO("BPWriter") as writer: + temps = writer.DefineVariable("temps", np.empty([4], dtype=np.int64)) + with writer.Open("pythontestvariable.bp", bindings.Mode.Write) as engine: + temps_measures = np.array([35, 40, 30, 45], dtype=np.int64) + engine.Put(temps, temps_measures) + + with adios.DeclareIO("BPReader") as reader: + with reader.Open("pythontestvariable.bp", bindings.Mode.Read) as engine: + engine.BeginStep() + temps = reader.InquireVariable("temps") + engine.EndStep() + + self.assertEqual(temps.Name(), "temps") + self.assertEqual(temps.BlockID(), 0) + self.assertEqual(temps.Count(), [0, 0]) + self.assertEqual(temps.Sizeof(), 8) + self.assertEqual(temps.Steps(), 1) + self.assertEqual(temps.StepsStart(), 0) + + def test_operators(self): + adios = ADIOS() + op1 = adios.DefineOperator("noop", "null") + with adios.DeclareIO("BPWriter") as writer: + temps = writer.DefineVariable("temps", np.empty([4], dtype=np.int64)) + temps.AddOperation(op1) + with writer.Open("pythontestvariable.bp", bindings.Mode.Write) as engine: + temps_measures = np.array([35, 40, 30, 45], dtype=np.int64) + engine.Put(temps, temps_measures) + + op2 = adios.DefineOperator("noop2", "null") + with adios.DeclareIO("BPReader") as reader: + with reader.Open("pythontestvariable.bp", bindings.Mode.Read) as engine: + engine.BeginStep() + temps = reader.InquireVariable("temps") + temps.AddOperation(op2) + engine.EndStep() + + +if __name__ == "__main__": + unittest.main()