[tmva][sofie] Disentangle PyMVA and SOFIE

cmake/modules/RootBuildOptions.cmake

@@ -175,7 +175,7 @@ ROOT_BUILD_OPTION(tmva-gpu OFF "Build TMVA with GPU support for deep learning (r
 ROOT_BUILD_OPTION(tmva-pymva OFF "Enable usage of Python ML libraries in TMVA (requires NumPy, works only with TensorFlow<=2.15)")
 ROOT_BUILD_OPTION(tmva-rmva OFF "Enable support for R in TMVA")
 ROOT_BUILD_OPTION(tmva-sofie OFF "Build TMVA with support for sofie - fast inference code generation (requires protobuf 3)")
-ROOT_BUILD_OPTION(tpython ON "Build the TPython class that allows you to run Python code from C++")
+ROOT_BUILD_OPTION(tpython ON "Build the TPython class that allows you to run Python code from C++, as well as other ROOT libraries that link against libpython")
 ROOT_BUILD_OPTION(unfold OFF "Enable the unfold package [GPL]")
 ROOT_BUILD_OPTION(unuran OFF "Enable support for UNURAN (package for generating non-uniform random numbers) [GPL]")
 ROOT_BUILD_OPTION(uring OFF "Enable support for io_uring (requires liburing and Linux kernel >= 5.1)")

cmake/modules/SearchInstalledSoftware.cmake

@@ -705,7 +705,7 @@ if(pyroot AND NOT (tpython OR tmva-pymva))
 elseif(tpython OR tmva-pymva)
   list(APPEND python_components Development)
 endif()
-if(tmva-pymva)
+if(tmva-pymva OR (tpython AND tmva))
   list(APPEND python_components NumPy)
 endif()
 find_package(Python3 3.9 COMPONENTS ${python_components})
@@ -1796,7 +1796,7 @@ if(tmva)
       endif()
     endif()
   endif()
-  if(tmva-pymva)
+  if(tmva-pymva OR (tpython AND tmva))
     if(fail-on-missing AND (NOT Python3_NumPy_FOUND OR NOT Python3_Development_FOUND))
       message(SEND_ERROR "TMVA: numpy python package or Python development package not found and tmva-pymva component required"
                           " (python executable: ${Python3_EXECUTABLE})")

tmva/pymva/CMakeLists.txt

@@ -17,17 +17,13 @@ ROOT_STANDARD_LIBRARY_PACKAGE(PyMVA
     TMVA/MethodPyKeras.h
     TMVA/MethodPyRandomForest.h
     TMVA/MethodPyTorch.h
-    TMVA/RModelParser_Keras.h
-    TMVA/RModelParser_PyTorch.h
     TMVA/PyMethodBase.h
   SOURCES
     src/MethodPyAdaBoost.cxx
     src/MethodPyGTB.cxx
     src/MethodPyKeras.cxx
     src/MethodPyRandomForest.cxx
     src/MethodPyTorch.cxx
-    src/RModelParser_Keras.cxx
-    src/RModelParser_PyTorch.cxx
     src/PyMethodBase.cxx
   LIBRARIES
     Python3::NumPy
@@ -38,7 +34,6 @@ ROOT_STANDARD_LIBRARY_PACKAGE(PyMVA
     Thread
     RIO
     TMVA
-    ROOTTMVASofie
 )
 
 ROOT_ADD_TEST_SUBDIRECTORY(test)

tmva/pymva/inc/LinkDef.h

@@ -15,8 +15,4 @@
 #pragma link C++ class TMVA::MethodPyGTB+;
 #pragma link C++ class TMVA::MethodPyKeras+;
 #pragma link C++ class TMVA::MethodPyTorch+;
-#pragma link C++ namespace TMVA::Experimental::SOFIE::PyKeras;
-#pragma link C++ function TMVA::Experimental::SOFIE::PyKeras::Parse+;
-#pragma link C++ namespace TMVA::Experimental::SOFIE::PyTorch;
-#pragma link C++ function TMVA::Experimental::SOFIE::PyTorch::Parse+;
 #endif

tmva/pymva/test/CMakeLists.txt

@@ -11,7 +11,7 @@
 
 project(pymva-tests)
 
-set(Libraries Core MathCore TMVA PyMVA ROOTTMVASofie)
+set(Libraries Core MathCore TMVA PyMVA)
 
 # Look for needed python modules
 ROOT_FIND_PYTHON_MODULE(torch)
@@ -55,10 +55,6 @@ endif(ROOT_SKLEARN_FOUND)
 
 # Enable tests based on available python modules
 if(ROOT_TORCH_FOUND)
-   configure_file(generatePyTorchModelClassification.py generatePyTorchModelClassification.py COPYONLY)
-   configure_file(generatePyTorchModelMulticlass.py generatePyTorchModelMulticlass.py COPYONLY)
-   configure_file(generatePyTorchModelRegression.py generatePyTorchModelRegression.py COPYONLY)
-   configure_file(generatePyTorchModels.py generatePyTorchModels.py COPYONLY)
    # Test PyTorch: Binary classification
 
    if (ROOT_SKLEARN_FOUND)
@@ -80,27 +76,9 @@ if(ROOT_TORCH_FOUND)
       LIBRARIES ${Libraries})
    ROOT_ADD_TEST(PyMVA-Torch-Multiclass COMMAND testPyTorchMulticlass DEPENDS ${PyMVA-Torch-Multiclass-depends})
 
-   # Test RModelParser_PyTorch
-
-   if(BLAS_FOUND)
-     ROOT_ADD_GTEST(TestRModelParserPyTorch TestRModelParserPyTorch.C
-           LIBRARIES
-           ROOTTMVASofie
-           TMVA
-           Python3::NumPy
-           Python3::Python
-           BLAS::BLAS
-           INCLUDE_DIRS
-           SYSTEM
-           ${CMAKE_CURRENT_BINARY_DIR}
-        )
-   endif()
-
 endif(ROOT_TORCH_FOUND)
 
 if((ROOT_KERAS_FOUND AND ROOT_THEANO_FOUND) OR (ROOT_KERAS_FOUND AND ROOT_TENSORFLOW_FOUND))
-   configure_file(generateKerasModels.py generateKerasModels.py COPYONLY)
-   configure_file(scale_by_2_op.hxx scale_by_2_op.hxx COPYONLY)
 
    if (ROOT_TORCH_FOUND)
       set(PyMVA-Keras-Classification-depends PyMVA-Torch-Classification)
@@ -127,19 +105,4 @@ if((ROOT_KERAS_FOUND AND ROOT_THEANO_FOUND) OR (ROOT_KERAS_FOUND AND ROOT_TENSOR
    ROOT_EXECUTABLE(testPyKerasMulticlass testPyKerasMulticlass.C
       LIBRARIES ${Libraries})
    ROOT_ADD_TEST(PyMVA-Keras-Multiclass COMMAND testPyKerasMulticlass DEPENDS ${PyMVA-Keras-Multiclass-depends})
-
-   if(BLAS_FOUND)
-      ROOT_ADD_GTEST(TestRModelParserKeras TestRModelParserKeras.C
-         LIBRARIES
-         ROOTTMVASofie
-         PyMVA
-         Python3::NumPy
-         Python3::Python
-         BLAS::BLAS
-         INCLUDE_DIRS
-         SYSTEM
-         ${CMAKE_CURRENT_BINARY_DIR}
-     )
-  endif()
-
 endif()

tmva/sofie/CMakeLists.txt

@@ -74,6 +74,9 @@ ROOT_STANDARD_LIBRARY_PACKAGE(ROOTTMVASofie
    TMVA/RFunction_MLP.hxx
    TMVA/RFunction_Sum.hxx
    TMVA/RFunction_Mean.hxx
+
+   TMVA/RModelParser_Keras.h
+   TMVA/RModelParser_PyTorch.h
   SOURCES
     src/RModel_Base.cxx
     src/RModel.cxx
@@ -90,6 +93,17 @@ ROOT_STANDARD_LIBRARY_PACKAGE(ROOTTMVASofie
     ${EXTRA_SOFIE_DEPENDENCIES}
 )
 
+if(tpython)
+  ROOT_LINKER_LIBRARY(ROOTTMVASofiePyParsers
+      src/RModelParser_Keras.cxx
+      src/RModelParser_PyTorch.cxx
+    LIBRARIES
+      Python3::NumPy
+      Python3::Python
+      ROOTTMVASofie
+  )
+endif()
+
 target_include_directories(ROOTTMVASofie PUBLIC
   $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>)
 set_target_properties(ROOTTMVASofie PROPERTIES

tmva/pymva/inc/TMVA/RModelParser_Keras.h → tmva/sofie/inc/TMVA/RModelParser_Keras.h

@@ -30,17 +30,11 @@
 #include "TMVA/Types.h"
 #include "TMVA/OperatorList.hxx"
 
-#include "TMVA/PyMethodBase.h"
-
 #include "Rtypes.h"
 #include "TString.h"
 
 
-namespace TMVA{
-namespace Experimental{
-namespace SOFIE{
-namespace PyKeras{
-
+namespace TMVA::Experimental::SOFIE::PyKeras {
 
 /// Parser function for translatng Keras .h5 model into a RModel object.
 /// Accepts the file location of a Keras model and returns the
@@ -49,8 +43,6 @@ namespace PyKeras{
 /// has not a defined input batch size : e.g. for input = (input_dim,)
 RModel Parse(std::string filename, int batch_size = -1);
 
-}//PyKeras
-}//SOFIE
-}//Experimental
-}//TMVA
+} // namespace TMVA::Experimental::SOFIE::PyKeras
+
 #endif //TMVA_PYMVA_RMODELPARSER_KERAS

tmva/pymva/inc/TMVA/RModelParser_PyTorch.h → tmva/sofie/inc/TMVA/RModelParser_PyTorch.h

@@ -30,31 +30,23 @@
 #include "TMVA/Types.h"
 #include "TMVA/OperatorList.hxx"
 
-#include "TMVA/PyMethodBase.h"
-
 #include "Rtypes.h"
 #include "TString.h"
 
 
-namespace TMVA{
-namespace Experimental{
-namespace SOFIE{
-namespace PyTorch{
+namespace TMVA::Experimental::SOFIE::PyTorch {
 
 /// Parser function for translating PyTorch .pt model into a RModel object.
 /// Accepts the file location of a PyTorch model, shapes and data-types of input tensors
 /// and returns the equivalent RModel object.
-RModel Parse(std::string filepath,std::vector<std::vector<size_t>> inputShapes, std::vector<ETensorType> dtype);
+RModel Parse(std::string filepath, std::vector<std::vector<size_t>> inputShapes, std::vector<ETensorType> dtype);
 
 /// Overloaded Parser function for translating PyTorch .pt model into a RModel object.
 /// Accepts the file location of a PyTorch model and the shapes of input tensors.
 /// Builds the vector of data-types for input tensors and calls the `Parse()` function to
 /// return the equivalent RModel object.
-RModel Parse(std::string filepath,std::vector<std::vector<size_t>> inputShapes);
+RModel Parse(std::string filepath, std::vector<std::vector<size_t>> inputShapes);
 
-}//PyTorch
-}//SOFIE
-}//Experimental
-}//TMVA
+} // namespace TMVA::Experimental::SOFIE::PyTorch
 
 #endif //TMVA_PYMVA_RMODELPARSER_PYTORCH

tmva/pymva/src/RModelParser_Keras.cxx → tmva/sofie/src/RModelParser_Keras.cxx

@@ -25,16 +25,48 @@
 #include <numpy/arrayobject.h>
 
 
-namespace TMVA{
-namespace Experimental{
-namespace SOFIE{
-namespace PyKeras{
+namespace TMVA::Experimental::SOFIE::PyKeras {
 
-// Referencing Python utility functions present in PyMethodBase
-static void(& PyRunString)(TString, PyObject*, PyObject*) = PyMethodBase::PyRunString;
-static const char*(& PyStringAsString)(PyObject*) = PyMethodBase::PyStringAsString;
-static std::vector<size_t>(& GetDataFromTuple)(PyObject*) = PyMethodBase::GetDataFromTuple;
-static PyObject*(& GetValueFromDict)(PyObject*, const char*) = PyMethodBase::GetValueFromDict;
+namespace {
+
+// Utility functions (taken from PyMethodBase in PyMVA)
+
+void PyRunString(TString code, PyObject *globalNS, PyObject *localNS)
+{
+   PyObject *fPyReturn = PyRun_String(code, Py_single_input, globalNS, localNS);
+   if (!fPyReturn) {
+      std::cout << "\nPython error message:\n";
+      PyErr_Print();
+      throw std::runtime_error("\nFailed to run python code: " + code);
+   }
+}
+
+const char *PyStringAsString(PyObject *string)
+{
+   PyObject *encodedString = PyUnicode_AsUTF8String(string);
+   const char *cstring = PyBytes_AsString(encodedString);
+   return cstring;
+}
+
+std::vector<size_t> GetDataFromTuple(PyObject *tupleObject)
+{
+   std::vector<size_t> tupleVec;
+   for (Py_ssize_t tupleIter = 0; tupleIter < PyTuple_Size(tupleObject); ++tupleIter) {
+      auto itemObj = PyTuple_GetItem(tupleObject, tupleIter);
+      if (itemObj == Py_None)
+         tupleVec.push_back(0); // case shape is for example (None,2,3)
+      else
+         tupleVec.push_back((size_t)PyLong_AsLong(itemObj));
+   }
+   return tupleVec;
+}
+
+PyObject *GetValueFromDict(PyObject *dict, const char *key)
+{
+   return PyDict_GetItemWithError(dict, PyUnicode_FromString(key));
+}
+
+} // namespace
 
 namespace INTERNAL{
 
@@ -1036,7 +1068,5 @@ RModel Parse(std::string filename, int batch_size){
 
    return rmodel;
 }
-}//PyKeras
-}//SOFIE
-}//Experimental
-}//TMVA
+
+} // namespace TMVA::Experimental::SOFIE::PyKeras

tmva/pymva/src/RModelParser_PyTorch.cxx → tmva/sofie/src/RModelParser_PyTorch.cxx

@@ -28,15 +28,39 @@
 #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
 #include <numpy/arrayobject.h>
 
-namespace TMVA{
-namespace Experimental{
-namespace SOFIE{
-namespace PyTorch{
+namespace TMVA::Experimental::SOFIE::PyTorch {
 
-// Referencing Python utility functions present in PyMethodBase
-static void(& PyRunString)(TString, PyObject*, PyObject*) = PyMethodBase::PyRunString;
-static const char*(& PyStringAsString)(PyObject*) = PyMethodBase::PyStringAsString;
-static std::vector<size_t>(& GetDataFromList)(PyObject*) = PyMethodBase::GetDataFromList;
+namespace {
+
+// Utility functions (taken from PyMethodBase in PyMVA)
+
+void PyRunString(TString code, PyObject *globalNS, PyObject *localNS)
+{
+   PyObject *fPyReturn = PyRun_String(code, Py_single_input, globalNS, localNS);
+   if (!fPyReturn) {
+      std::cout << "\nPython error message:\n";
+      PyErr_Print();
+      throw std::runtime_error("\nFailed to run python code: " + code);
+   }
+}
+
+const char *PyStringAsString(PyObject *string)
+{
+   PyObject *encodedString = PyUnicode_AsUTF8String(string);
+   const char *cstring = PyBytes_AsString(encodedString);
+   return cstring;
+}
+
+std::vector<size_t> GetDataFromList(PyObject *listObject)
+{
+   std::vector<size_t> listVec;
+   for (Py_ssize_t listIter = 0; listIter < PyList_Size(listObject); ++listIter) {
+      listVec.push_back((size_t)PyLong_AsLong(PyList_GetItem(listObject, listIter)));
+   }
+   return listVec;
+}
+
+} // namespace
 
 
 namespace INTERNAL{
@@ -558,7 +582,5 @@ RModel Parse(std::string filepath,std::vector<std::vector<size_t>> inputShapes){
       std::vector<ETensorType> dtype(inputShapes.size(),ETensorType::FLOAT);
       return Parse(filepath,inputShapes,dtype);
 }
-}//PyTorch
-}//SOFIE
-}//Experimental
-}//TMVA
+
+} // namespace TMVA::Experimental::SOFIE::PyTorch