Skip to content

Commit 4b9fc94

Browse files
pbuehlershahor02
pbuehler
authored and
shahor02
committed
Add aod-writer options resdir and maxfilesize as command line options
1 parent 7a34416 commit 4b9fc94

File tree

6 files changed

+45
-15
lines changed

6 files changed

+45
-15
lines changed

‎Framework/Core/include/Framework/DataOutputDirector.h‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -74,8 +74,8 @@ struct DataOutputDirector {
7474
void readSpecs(std::vector<InputSpec> inputs);
7575

7676
// fill the DataOutputDirector with information from a json file
77-
std::tuple<std::string, std::string, int> readJson(std::string const& fnjson);
78-
std::tuple<std::string, std::string, int> readJsonString(std::string const& stjson);
77+
std::tuple<std::string, std::string, std::string, float, int> readJson(std::string const& fnjson);
78+
std::tuple<std::string, std::string, std::string, float, int> readJsonString(std::string const& stjson);
7979

8080
// read/write private members
8181
int getNumberTimeFramesToMerge() { return mnumberTimeFramesToMerge; }
@@ -117,8 +117,8 @@ struct DataOutputDirector {
117117
int mnumberTimeFramesToMerge = 1;
118118
std::string mfileMode = "RECREATE";
119119

120-
std::tuple<std::string, std::string, int> readJsonDocument(Document* doc);
121-
const std::tuple<std::string, std::string, int> memptyanswer = std::make_tuple(std::string(""), std::string(""), -1);
120+
std::tuple<std::string, std::string, std::string, float, int> readJsonDocument(Document* doc);
121+
const std::tuple<std::string, std::string, std::string, float, int> memptyanswer = std::make_tuple(std::string(""), std::string(""), std::string(""), -1., -1);
122122
};
123123

124124
} // namespace o2::framework

‎Framework/Core/src/DataOutputDirector.cxx‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -224,7 +224,7 @@ void DataOutputDirector::readSpecs(std::vector<InputSpec> inputs)
224224
}
225225
}
226226

227-
std::tuple<std::string, std::string, int> DataOutputDirector::readJson(std::string const& fnjson)
227+
std::tuple<std::string, std::string, std::string, float, int> DataOutputDirector::readJson(std::string const& fnjson)
228228
{
229229
// open the file
230230
FILE* fjson = fopen(fnjson.c_str(), "r");
@@ -240,25 +240,25 @@ std::tuple<std::string, std::string, int> DataOutputDirector::readJson(std::stri
240240
// parse the json file
241241
Document jsonDocument;
242242
jsonDocument.ParseStream(jsonStream);
243-
auto [dfn, fmode, ntfm] = readJsonDocument(&jsonDocument);
243+
auto [rdn, dfn, fmode, mfs, ntfm] = readJsonDocument(&jsonDocument);
244244

245245
// clean up
246246
fclose(fjson);
247247

248-
return std::make_tuple(dfn, fmode, ntfm);
248+
return std::make_tuple(rdn, dfn, fmode, mfs, ntfm);
249249
}
250250

251-
std::tuple<std::string, std::string, int> DataOutputDirector::readJsonString(std::string const& jsonString)
251+
std::tuple<std::string, std::string, std::string, float, int> DataOutputDirector::readJsonString(std::string const& jsonString)
252252
{
253253
// parse the json string
254254
Document jsonDocument;
255255
jsonDocument.Parse(jsonString.c_str());
256-
auto [dfn, fmode, ntfm] = readJsonDocument(&jsonDocument);
256+
auto [rdn, dfn, fmode, mfs, ntfm] = readJsonDocument(&jsonDocument);
257257

258-
return std::make_tuple(dfn, fmode, ntfm);
258+
return std::make_tuple(rdn, dfn, fmode, mfs, ntfm);
259259
}
260260

261-
std::tuple<std::string, std::string, int> DataOutputDirector::readJsonDocument(Document* jsonDocument)
261+
std::tuple<std::string, std::string, std::string, float, int> DataOutputDirector::readJsonDocument(Document* jsonDocument)
262262
{
263263
std::string smc(":");
264264
std::string slh("/");
@@ -428,7 +428,7 @@ std::tuple<std::string, std::string, int> DataOutputDirector::readJsonDocument(D
428428
printOut();
429429
}
430430

431-
return std::make_tuple(dfn, fmode, ntfm);
431+
return std::make_tuple(resdir, dfn, fmode, maxfs, ntfm);
432432
}
433433

434434
std::vector<DataOutputDescriptor*> DataOutputDirector::getDataOutputDescriptors(header::DataHeader dh)
@@ -572,7 +572,9 @@ void DataOutputDirector::closeDataFiles()
572572
void DataOutputDirector::printOut()
573573
{
574574
LOGP(info, "DataOutputDirector");
575+
LOGP(info, " Output directory : {}", mresultDirectory);
575576
LOGP(info, " Default file name : {}", mfilenameBase);
577+
LOGP(info, " Maximum file size : {} megabytes", mmaxfilesize);
576578
LOGP(info, " Number of files : {}", mfilenameBases.size());
577579

578580
LOGP(info, " DataOutputDescriptors: {}", mDataOutputDescriptors.size());

‎Framework/Core/src/WorkflowCustomizationHelpers.cxx‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,9 @@ std::vector<ConfigParamSpec> WorkflowCustomizationHelpers::requiredWorkflowOptio
5656

5757
// options for AOD writer
5858
{"aod-writer-json", VariantType::String, "", {"Name of the json configuration file"}},
59+
{"aod-writer-resdir", VariantType::String, "", {"Name of the output directory"}},
5960
{"aod-writer-resfile", VariantType::String, "", {"Default name of the output file"}},
61+
{"aod-writer-maxfilesize", VariantType::Float, 0.0f, {"Maximum size of an output file in megabytes"}},
6062
{"aod-writer-resmode", VariantType::String, "RECREATE", {"Creation mode of the result files: NEW, CREATE, RECREATE, UPDATE"}},
6163
{"aod-writer-ntfmerge", VariantType::Int, -1, {"Number of time frames to merge into one file"}},
6264
{"aod-writer-keep", VariantType::String, "", {"Comma separated list of ORIGIN/DESCRIPTION/SUBSPECIFICATION:treename:col1/col2/..:filename"}},

‎Framework/Core/src/WorkflowHelpers.cxx‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -973,28 +973,42 @@ std::shared_ptr<DataOutputDirector> WorkflowHelpers::getDataOutputDirector(Confi
973973

974974
// analyze options and take actions accordingly
975975
// default values
976+
std::string rdn, resdir("./");
976977
std::string fnb, fnbase("AnalysisResults_trees");
978+
float mfs, maxfilesize(-1.);
977979
std::string fmo, filemode("RECREATE");
978980
int ntfm, ntfmerge = 1;
979981

980982
// values from json
981983
if (options.isSet("aod-writer-json")) {
982984
auto fnjson = options.get<std::string>("aod-writer-json");
983985
if (!fnjson.empty()) {
984-
std::tie(fnb, fmo, ntfm) = dod->readJson(fnjson);
986+
std::tie(rdn, fnb, fmo, mfs, ntfm) = dod->readJson(fnjson);
987+
if (!rdn.empty()) {
988+
resdir = rdn;
989+
}
985990
if (!fnb.empty()) {
986991
fnbase = fnb;
987992
}
988993
if (!fmo.empty()) {
989994
filemode = fmo;
990995
}
996+
if (mfs > 0.) {
997+
maxfilesize = mfs;
998+
}
991999
if (ntfm > 0) {
9921000
ntfmerge = ntfm;
9931001
}
9941002
}
9951003
}
9961004

9971005
// values from command line options, information from json is overwritten
1006+
if (options.isSet("aod-writer-resdir")) {
1007+
rdn = options.get<std::string>("aod-writer-resdir");
1008+
if (!rdn.empty()) {
1009+
resdir = rdn;
1010+
}
1011+
}
9981012
if (options.isSet("aod-writer-resfile")) {
9991013
fnb = options.get<std::string>("aod-writer-resfile");
10001014
if (!fnb.empty()) {
@@ -1007,6 +1021,12 @@ std::shared_ptr<DataOutputDirector> WorkflowHelpers::getDataOutputDirector(Confi
10071021
filemode = fmo;
10081022
}
10091023
}
1024+
if (options.isSet("aod-writer-maxfilesize")) {
1025+
mfs = options.get<float>("aod-writer-maxfilesize");
1026+
if (mfs > 0) {
1027+
maxfilesize = mfs;
1028+
}
1029+
}
10101030
if (options.isSet("aod-writer-ntfmerge")) {
10111031
ntfm = options.get<int>("aod-writer-ntfmerge");
10121032
if (ntfm > 0) {
@@ -1039,8 +1059,10 @@ std::shared_ptr<DataOutputDirector> WorkflowHelpers::getDataOutputDirector(Confi
10391059
}
10401060
}
10411061
}
1062+
dod->setResultDir(resdir);
10421063
dod->setFilenameBase(fnbase);
10431064
dod->setFileMode(filemode);
1065+
dod->setMaximumFileSize(maxfilesize);
10441066
dod->setNumberTimeFramesToMerge(ntfmerge);
10451067

10461068
return dod;

‎Framework/Core/src/runDataProcessing.cxx‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1803,8 +1803,10 @@ int runStateMachine(DataProcessorSpecs const& workflow,
18031803
"--aod-memory-rate-limit",
18041804
"--aod-writer-json",
18051805
"--aod-writer-ntfmerge",
1806+
"--aod-writer-resdir",
18061807
"--aod-writer-resfile",
18071808
"--aod-writer-resmode",
1809+
"--aod-writer-maxfilesize",
18081810
"--aod-writer-keep",
18091811
"--aod-parent-access-level",
18101812
"--aod-parent-base-path-replacement",

‎Framework/Core/test/test_DataOutputDirector.cxx‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,10 @@ BOOST_AUTO_TEST_CASE(TestDataOutputDirector)
4949
BOOST_CHECK_EQUAL(ds[1]->getFilenameBase(), std::string("myresultfile"));
5050

5151
// test jsonString reader
52+
std::string rdn("./");
5253
std::string dfn("");
5354
std::string fmode("");
55+
float mfs = -1.;
5456
int ntf = -1;
5557

5658
dh = DataHeader(DataDescription{"DUE"},
@@ -59,7 +61,7 @@ BOOST_AUTO_TEST_CASE(TestDataOutputDirector)
5961
std::string jsonString(R"({"OutputDirector": {"resfile": "defresults", "resfilemode": "RECREATE", "ntfmerge": 10, "OutputDescriptors": [{"table": "AOD/UNO/0", "columns": ["fEta1","fMom1"], "treename": "uno", "filename": "unoresults"}, {"table": "AOD/DUE/0", "columns": ["fPhi2"], "treename": "due"}]}})");
6062

6163
dod.reset();
62-
std::tie(dfn, fmode, ntf) = dod.readJsonString(jsonString);
64+
std::tie(rdn, dfn, fmode, mfs, ntf) = dod.readJsonString(jsonString);
6365
//dod.printOut(); printf("\n\n");
6466
ds = dod.getDataOutputDescriptors(dh);
6567

@@ -105,7 +107,7 @@ BOOST_AUTO_TEST_CASE(TestDataOutputDirector)
105107
jf.close();
106108

107109
dod.reset();
108-
std::tie(dfn, fmode, ntf) = dod.readJson(jsonFile);
110+
std::tie(rdn, dfn, fmode, mfs, ntf) = dod.readJson(jsonFile);
109111
dod.setFilenameBase("AnalysisResults");
110112
//dod.printOut(); printf("\n\n");
111113
ds = dod.getDataOutputDescriptors(dh);

0 commit comments

Comments
 (0)