Skip to content

Commit 8a1bbdc

Browse files
guitargeekguitargeek
committed
[TMVA] Disentangle PyMVA and SOFIE
So far, the part of SOFIE that uses the C Python API was included in PyMVA for convenience, but the functionality is completely unrelated. Moving this code to SOFIE itself means we can fully build and test SOFIE without enabling `tmva-pymva`. The only subtelty is that we want to be able to disable these SOFIE features that require linking against `libpython`, because this is not always allowed (e.g. in the Python wheels). Therefore, this part is only built if we build also the other ROOT libraries that links against `libpython` besides PyMVA, wich is `TPython`.
1 parent 4c545d7 commit 8a1bbdc

21 files changed

+182
-105
lines changed

‎cmake/modules/RootBuildOptions.cmake‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -175,7 +175,7 @@ ROOT_BUILD_OPTION(tmva-gpu OFF "Build TMVA with GPU support for deep learning (r
175175
ROOT_BUILD_OPTION(tmva-pymva OFF "Enable usage of Python ML libraries in TMVA (requires NumPy, works only with TensorFlow<=2.15)")
176176
ROOT_BUILD_OPTION(tmva-rmva OFF "Enable support for R in TMVA")
177177
ROOT_BUILD_OPTION(tmva-sofie OFF "Build TMVA with support for sofie - fast inference code generation (requires protobuf 3)")
178-
ROOT_BUILD_OPTION(tpython ON "Build the TPython class that allows you to run Python code from C++")
178+
ROOT_BUILD_OPTION(tpython ON "Build the TPython class that allows you to run Python code from C++, as well as other ROOT libraries that link against libpython")
179179
ROOT_BUILD_OPTION(unfold OFF "Enable the unfold package [GPL]")
180180
ROOT_BUILD_OPTION(unuran OFF "Enable support for UNURAN (package for generating non-uniform random numbers) [GPL]")
181181
ROOT_BUILD_OPTION(uring OFF "Enable support for io_uring (requires liburing and Linux kernel >= 5.1)")

‎cmake/modules/SearchInstalledSoftware.cmake‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -705,7 +705,7 @@ if(pyroot AND NOT (tpython OR tmva-pymva))
705705
elseif(tpython OR tmva-pymva)
706706
list(APPEND python_components Development)
707707
endif()
708-
if(tmva-pymva)
708+
if(tmva-pymva OR (tpython AND tmva))
709709
list(APPEND python_components NumPy)
710710
endif()
711711
find_package(Python3 3.9 COMPONENTS ${python_components})
@@ -1796,7 +1796,7 @@ if(tmva)
17961796
endif()
17971797
endif()
17981798
endif()
1799-
if(tmva-pymva)
1799+
if(tmva-pymva OR (tpython AND tmva))
18001800
if(fail-on-missing AND (NOT Python3_NumPy_FOUND OR NOT Python3_Development_FOUND))
18011801
message(SEND_ERROR "TMVA: numpy python package or Python development package not found and tmva-pymva component required"
18021802
" (python executable: ${Python3_EXECUTABLE})")

‎tmva/pymva/CMakeLists.txt‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,17 +17,13 @@ ROOT_STANDARD_LIBRARY_PACKAGE(PyMVA
1717
TMVA/MethodPyKeras.h
1818
TMVA/MethodPyRandomForest.h
1919
TMVA/MethodPyTorch.h
20-
TMVA/RModelParser_Keras.h
21-
TMVA/RModelParser_PyTorch.h
2220
TMVA/PyMethodBase.h
2321
SOURCES
2422
src/MethodPyAdaBoost.cxx
2523
src/MethodPyGTB.cxx
2624
src/MethodPyKeras.cxx
2725
src/MethodPyRandomForest.cxx
2826
src/MethodPyTorch.cxx
29-
src/RModelParser_Keras.cxx
30-
src/RModelParser_PyTorch.cxx
3127
src/PyMethodBase.cxx
3228
LIBRARIES
3329
Python3::NumPy
@@ -38,7 +34,6 @@ ROOT_STANDARD_LIBRARY_PACKAGE(PyMVA
3834
Thread
3935
RIO
4036
TMVA
41-
ROOTTMVASofie
4237
)
4338

4439
ROOT_ADD_TEST_SUBDIRECTORY(test)

‎tmva/pymva/inc/LinkDef.h‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,4 @@
1515
#pragma link C++ class TMVA::MethodPyGTB+;
1616
#pragma link C++ class TMVA::MethodPyKeras+;
1717
#pragma link C++ class TMVA::MethodPyTorch+;
18-
#pragma link C++ namespace TMVA::Experimental::SOFIE::PyKeras;
19-
#pragma link C++ function TMVA::Experimental::SOFIE::PyKeras::Parse+;
20-
#pragma link C++ namespace TMVA::Experimental::SOFIE::PyTorch;
21-
#pragma link C++ function TMVA::Experimental::SOFIE::PyTorch::Parse+;
2218
#endif

‎tmva/pymva/test/CMakeLists.txt‎

Lines changed: 1 addition & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111

1212
project(pymva-tests)
1313

14-
set(Libraries Core MathCore TMVA PyMVA ROOTTMVASofie)
14+
set(Libraries Core MathCore TMVA PyMVA)
1515

1616
# Look for needed python modules
1717
ROOT_FIND_PYTHON_MODULE(torch)
@@ -55,10 +55,6 @@ endif(ROOT_SKLEARN_FOUND)
5555

5656
# Enable tests based on available python modules
5757
if(ROOT_TORCH_FOUND)
58-
configure_file(generatePyTorchModelClassification.py generatePyTorchModelClassification.py COPYONLY)
59-
configure_file(generatePyTorchModelMulticlass.py generatePyTorchModelMulticlass.py COPYONLY)
60-
configure_file(generatePyTorchModelRegression.py generatePyTorchModelRegression.py COPYONLY)
61-
configure_file(generatePyTorchModels.py generatePyTorchModels.py COPYONLY)
6258
# Test PyTorch: Binary classification
6359

6460
if (ROOT_SKLEARN_FOUND)
@@ -80,27 +76,9 @@ if(ROOT_TORCH_FOUND)
8076
LIBRARIES ${Libraries})
8177
ROOT_ADD_TEST(PyMVA-Torch-Multiclass COMMAND testPyTorchMulticlass DEPENDS ${PyMVA-Torch-Multiclass-depends})
8278

83-
# Test RModelParser_PyTorch
84-
85-
if(BLAS_FOUND)
86-
ROOT_ADD_GTEST(TestRModelParserPyTorch TestRModelParserPyTorch.C
87-
LIBRARIES
88-
ROOTTMVASofie
89-
TMVA
90-
Python3::NumPy
91-
Python3::Python
92-
BLAS::BLAS
93-
INCLUDE_DIRS
94-
SYSTEM
95-
${CMAKE_CURRENT_BINARY_DIR}
96-
)
97-
endif()
98-
9979
endif(ROOT_TORCH_FOUND)
10080

10181
if((ROOT_KERAS_FOUND AND ROOT_THEANO_FOUND) OR (ROOT_KERAS_FOUND AND ROOT_TENSORFLOW_FOUND))
102-
configure_file(generateKerasModels.py generateKerasModels.py COPYONLY)
103-
configure_file(scale_by_2_op.hxx scale_by_2_op.hxx COPYONLY)
10482

10583
if (ROOT_TORCH_FOUND)
10684
set(PyMVA-Keras-Classification-depends PyMVA-Torch-Classification)
@@ -127,19 +105,4 @@ if((ROOT_KERAS_FOUND AND ROOT_THEANO_FOUND) OR (ROOT_KERAS_FOUND AND ROOT_TENSOR
127105
ROOT_EXECUTABLE(testPyKerasMulticlass testPyKerasMulticlass.C
128106
LIBRARIES ${Libraries})
129107
ROOT_ADD_TEST(PyMVA-Keras-Multiclass COMMAND testPyKerasMulticlass DEPENDS ${PyMVA-Keras-Multiclass-depends})
130-
131-
if(BLAS_FOUND)
132-
ROOT_ADD_GTEST(TestRModelParserKeras TestRModelParserKeras.C
133-
LIBRARIES
134-
ROOTTMVASofie
135-
PyMVA
136-
Python3::NumPy
137-
Python3::Python
138-
BLAS::BLAS
139-
INCLUDE_DIRS
140-
SYSTEM
141-
${CMAKE_CURRENT_BINARY_DIR}
142-
)
143-
endif()
144-
145108
endif()

‎tmva/sofie/CMakeLists.txt‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,10 @@ if(sofie_root_support)
1717
list(APPEND EXTRA_SOFIE_DEPENDENCIES RIO)
1818
endif()
1919

20+
if(tpython)
21+
list(APPEND EXTRA_SOFIE_LIBRARIES Python3::NumPy Python3::Python)
22+
endif()
23+
2024
ROOT_STANDARD_LIBRARY_PACKAGE(ROOTTMVASofie
2125
HEADERS
2226
TMVA/OperatorList.hxx
@@ -74,6 +78,9 @@ ROOT_STANDARD_LIBRARY_PACKAGE(ROOTTMVASofie
7478
TMVA/RFunction_MLP.hxx
7579
TMVA/RFunction_Sum.hxx
7680
TMVA/RFunction_Mean.hxx
81+
82+
TMVA/RModelParser_Keras.h
83+
TMVA/RModelParser_PyTorch.h
7784
SOURCES
7885
src/RModel_Base.cxx
7986
src/RModel.cxx
@@ -84,6 +91,10 @@ ROOT_STANDARD_LIBRARY_PACKAGE(ROOTTMVASofie
8491
src/RFunction_Mean.cxx
8592
src/RFunction_Sum.cxx
8693
src/SOFIE_common.cxx
94+
src/RModelParser_Keras.cxx
95+
src/RModelParser_PyTorch.cxx
96+
LIBRARIES
97+
${EXTRA_SOFIE_LIBRARIES}
8798
DEPENDENCIES
8899
Core
89100
TMVA

‎tmva/sofie/inc/LinkDef.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,9 @@
1717
#pragma link C++ struct TMVA::Experimental::SOFIE::Dim+;
1818
#pragma link C++ struct TMVA::Experimental::SOFIE::GNN_Data+;
1919

20+
#pragma link C++ namespace TMVA::Experimental::SOFIE::PyKeras;
21+
#pragma link C++ function TMVA::Experimental::SOFIE::PyKeras::Parse+;
22+
#pragma link C++ namespace TMVA::Experimental::SOFIE::PyTorch;
23+
#pragma link C++ function TMVA::Experimental::SOFIE::PyTorch::Parse+;
24+
2025
#endif

‎tmva/pymva/inc/TMVA/RModelParser_Keras.h‎ renamed to ‎tmva/sofie/inc/TMVA/RModelParser_Keras.h‎

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -30,17 +30,11 @@
3030
#include "TMVA/Types.h"
3131
#include "TMVA/OperatorList.hxx"
3232

33-
#include "TMVA/PyMethodBase.h"
34-
3533
#include "Rtypes.h"
3634
#include "TString.h"
3735

3836

39-
namespace TMVA{
40-
namespace Experimental{
41-
namespace SOFIE{
42-
namespace PyKeras{
43-
37+
namespace TMVA::Experimental::SOFIE::PyKeras {
4438

4539
/// Parser function for translatng Keras .h5 model into a RModel object.
4640
/// Accepts the file location of a Keras model and returns the
@@ -49,8 +43,6 @@ namespace PyKeras{
4943
/// has not a defined input batch size : e.g. for input = (input_dim,)
5044
RModel Parse(std::string filename, int batch_size = -1);
5145

52-
}//PyKeras
53-
}//SOFIE
54-
}//Experimental
55-
}//TMVA
46+
} // namespace TMVA::Experimental::SOFIE::PyKeras
47+
5648
#endif //TMVA_PYMVA_RMODELPARSER_KERAS

‎tmva/pymva/inc/TMVA/RModelParser_PyTorch.h‎ renamed to ‎tmva/sofie/inc/TMVA/RModelParser_PyTorch.h‎

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -30,31 +30,23 @@
3030
#include "TMVA/Types.h"
3131
#include "TMVA/OperatorList.hxx"
3232

33-
#include "TMVA/PyMethodBase.h"
34-
3533
#include "Rtypes.h"
3634
#include "TString.h"
3735

3836

39-
namespace TMVA{
40-
namespace Experimental{
41-
namespace SOFIE{
42-
namespace PyTorch{
37+
namespace TMVA::Experimental::SOFIE::PyTorch {
4338

4439
/// Parser function for translating PyTorch .pt model into a RModel object.
4540
/// Accepts the file location of a PyTorch model, shapes and data-types of input tensors
4641
/// and returns the equivalent RModel object.
47-
RModel Parse(std::string filepath,std::vector<std::vector<size_t>> inputShapes, std::vector<ETensorType> dtype);
42+
RModel Parse(std::string filepath, std::vector<std::vector<size_t>> inputShapes, std::vector<ETensorType> dtype);
4843

4944
/// Overloaded Parser function for translating PyTorch .pt model into a RModel object.
5045
/// Accepts the file location of a PyTorch model and the shapes of input tensors.
5146
/// Builds the vector of data-types for input tensors and calls the `Parse()` function to
5247
/// return the equivalent RModel object.
53-
RModel Parse(std::string filepath,std::vector<std::vector<size_t>> inputShapes);
48+
RModel Parse(std::string filepath, std::vector<std::vector<size_t>> inputShapes);
5449

55-
}//PyTorch
56-
}//SOFIE
57-
}//Experimental
58-
}//TMVA
50+
} // namespace TMVA::Experimental::SOFIE::PyTorch
5951

6052
#endif //TMVA_PYMVA_RMODELPARSER_PYTORCH

‎tmva/pymva/src/RModelParser_Keras.cxx‎ renamed to ‎tmva/sofie/src/RModelParser_Keras.cxx‎

Lines changed: 51 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -19,22 +19,62 @@
1919

2020
#include "TMVA/RModelParser_Keras.h"
2121

22+
#include "TMVA/Tools.h"
23+
#include "TMVA/MethodBase.h"
24+
#include "TMVA/Types.h"
25+
26+
#include "Rtypes.h"
27+
#include "TString.h"
28+
#include <vector>
29+
2230
#include <Python.h>
2331

2432
#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
2533
#include <numpy/arrayobject.h>
2634

2735

28-
namespace TMVA{
29-
namespace Experimental{
30-
namespace SOFIE{
31-
namespace PyKeras{
36+
namespace TMVA::Experimental::SOFIE::PyKeras {
37+
38+
namespace {
39+
40+
// Utility functions (taken from PyMethodBase in PyMVA)
41+
42+
void PyRunString(TString code, PyObject *globalNS, PyObject *localNS)
43+
{
44+
PyObject *fPyReturn = PyRun_String(code, Py_single_input, globalNS, localNS);
45+
if (!fPyReturn) {
46+
std::cout << "\nPython error message:\n";
47+
PyErr_Print();
48+
throw std::runtime_error("\nFailed to run python code: " + code);
49+
}
50+
}
3251

33-
// Referencing Python utility functions present in PyMethodBase
34-
static void(& PyRunString)(TString, PyObject*, PyObject*) = PyMethodBase::PyRunString;
35-
static const char*(& PyStringAsString)(PyObject*) = PyMethodBase::PyStringAsString;
36-
static std::vector<size_t>(& GetDataFromTuple)(PyObject*) = PyMethodBase::GetDataFromTuple;
37-
static PyObject*(& GetValueFromDict)(PyObject*, const char*) = PyMethodBase::GetValueFromDict;
52+
const char *PyStringAsString(PyObject *string)
53+
{
54+
PyObject *encodedString = PyUnicode_AsUTF8String(string);
55+
const char *cstring = PyBytes_AsString(encodedString);
56+
return cstring;
57+
}
58+
59+
std::vector<size_t> GetDataFromTuple(PyObject *tupleObject)
60+
{
61+
std::vector<size_t> tupleVec;
62+
for (Py_ssize_t tupleIter = 0; tupleIter < PyTuple_Size(tupleObject); ++tupleIter) {
63+
auto itemObj = PyTuple_GetItem(tupleObject, tupleIter);
64+
if (itemObj == Py_None)
65+
tupleVec.push_back(0); // case shape is for example (None,2,3)
66+
else
67+
tupleVec.push_back((size_t)PyLong_AsLong(itemObj));
68+
}
69+
return tupleVec;
70+
}
71+
72+
PyObject *GetValueFromDict(PyObject *dict, const char *key)
73+
{
74+
return PyDict_GetItemWithError(dict, PyUnicode_FromString(key));
75+
}
76+
77+
} // namespace
3878

3979
namespace INTERNAL{
4080

@@ -1036,7 +1076,5 @@ RModel Parse(std::string filename, int batch_size){
10361076

10371077
return rmodel;
10381078
}
1039-
}//PyKeras
1040-
}//SOFIE
1041-
}//Experimental
1042-
}//TMVA
1079+
1080+
} // namespace TMVA::Experimental::SOFIE::PyKeras

0 commit comments

Comments
 (0)