fea: Merge mu/ele workflows and several updates

README.md

@@ -494,12 +494,15 @@ You can specify `-v all` to plot all the variables in the `coffea` file, or use
 
 :new: non-uniform rebinning is possible, specify the bins with  list of edges `--autorebin 50,80,81,82,83,100.5`
 
+```bash
+python scripts/plotdataMC.py -i a.coffea,b.coffea --lumi 41500 -p ttdilep_sf -v z_mass,z_pt  
+python scripts/plotdataMC.py -i "test*.coffea" --lumi 41500 -p ttdilep_sf -v z_mass,z_pt # with wildcard option need ""
+```
+
 <details><summary>more arguments</summary>
 <p>
 
 ```
-python plotdataMC.py -i a.coffea,b.coffea --lumi 41500 -p ttdilep_sf -v z_mass,z_pt 
-python plotdataMC.py -i "test*.coffea" --lumi 41500 -p ttdilep_sf -v z_mass,z_pt 
 
 options:
   -h, --help            show this help message and exit
@@ -537,13 +540,18 @@ options:
 You can specify `-v all` to plot all the variables in the `coffea` file, or use wildcard options (e.g. `-v "*DeepJet*"` for the input variables containing `DeepJet`)
 :exclamation_mark: If using wildcard for input, do not forget the quoatation marks! (see 2nd example below)
 
-<details><summary>more arguments</summary>
-<p>
+```bash
+# with merge map, compare ttbar with data
+python scripts/comparison.py -i "*.coffea" --mergemap '{"ttbar": ["TTto2L2Nu_TuneCP5_13p6TeV_powheg-pythia8","TTto4Q_TuneCP5_13p6TeV_powheg-pythia8","TTtoLNu2Q_TuneCP5_13p6TeV_powheg-pythia8],"data":["MuonRun2022C-27Jun2023-v1","MuonRun2022D-27Jun2023-v1"]}' -r ttbar -c data -v mu_pt  -p ttdilep_sf
+# if no  mergemap, take the key name directly
+python scripts/comparison.py -i datac.coffea,datad.coffea -r MuonRun2022C-27Jun2023-v1 -c MuonRun2022D-27Jun2023-v1 -v mu_pt  -p ttdilep_sf
 
 ```
-python comparison.py -i a.coffea,b.coffea -p ttsemilep_sf -r SingleMuon_Run2017B-106X_PFNanov1 -c DYJetsToLL_M-50_TuneCP5_13TeV-madgraphMLM-pythia8 -v DeepJet_Cpfcan_BtagPf_trackJetDistVal_0 --shortref Run2017B --shortcomp DYJets (--sepflav True/False)
-python comparison.py -i "test*.coffea" -p ttsemilep_sf -r SingleMuon_Run2017B-106X_PFNanov1 -c DYJetsToLL_M-50_TuneCP5_13TeV-madgraphMLM-pythia8 -v DeepJet_Cpfcan_BtagPf_trackJetDistVal_0 --shortref Run2017B --shortcomp DYJets (--sepflav True/False)
 
+<details><summary>more arguments</summary>
+<p>
+
+
 options:
   -h, --help            show this help message and exit
   -p {dilep_sf,ttsemilep_sf,ctag_Wc_sf,ctag_DY_sf,ctag_ttsemilep_sf,ctag_ttdilep_sf}, --phase {dilep_sf,ttsemilep_sf,ctag_Wc_sf,ctag_DY_sf,ctag_ttsemilep_sf,ctag_ttdilep_sf}
@@ -559,9 +567,8 @@ options:
                         variables to plot, splitted by ,. Wildcard option * available as well. Specifying `all` will run through all variables.
   --ext EXT             prefix name
   --com COM             sqrt(s) in TeV
-  --shortref SHORTREF   short name for reference dataset for legend
-  --shortcomp SHORTCOMP
-                        short names for compared datasets for legend, split by ','
+  --mergemap MERGEMAP
+                        Group list of sample(keys in coffea) as reference/compare set as dictionary format. Keys would be the new lables of the group
    --autorebin AUTOREBIN
                         Rebin the plotting variables, input `int` or `list`. int: merge N bins. list of number: rebin edges(non-uniform bin is possible)
    --xlabel XLABEL      rename the label for x-axis
@@ -654,4 +661,3 @@ Host *_f
 jupyter notebook --ip=127.0.0.1 --port 8800 --no-browser
 ```
 4. URL for notebook will be printed, copy and open in local browser
-

condor/execute.sh

@@ -32,7 +32,7 @@ fi
 
 # Get arguments
 declare -A ARGS
-for key in workflow output samplejson year campaign isCorr isSyst isJERC isArray noHist overwrite voms chunk retries outputXrootdDir remoteRepo; do
+for key in workflow output samplejson year campaign isSyst isArray noHist overwrite voms chunk skipbadfiles outputXrootdDir remoteRepo; do
     ARGS[$key]=$(jq -r ".$key" $WORKDIR/arguments.json)
 done
 
@@ -58,11 +58,11 @@ pip install psutil
 python -c "import json; json.dump(json.load(open('$WORKDIR/split_samples.json'))['$JOBID'], open('sample.json', 'w'), indent=4)"
 
 # Unparse arguments and send to runner.py
-OPTS="--wf ${ARGS[workflow]} --year ${ARGS[year]} --campaign ${ARGS[campaign]} --chunk ${ARGS[chunk]} --retries ${ARGS[retries]}"
+OPTS="--wf ${ARGS[workflow]} --year ${ARGS[year]} --campaign ${ARGS[campaign]} --chunk ${ARGS[chunk]}"
 if [ "${ARGS[voms]}" != "null" ]; then
     OPTS="$OPTS --voms ${ARGS[voms]}"
 fi
-for key in isCorr isSyst isJERC isArray noHist overwrite; do
+for key in  isSyst isArray noHist overwrite skipbadfiles; do
     if [ "${ARGS[$key]}" == true ]; then
         OPTS="$OPTS --$key"
     fi

condor/submitter.py

@@ -81,17 +81,13 @@ def get_main_parser():
         ],
         help="Dataset campaign, change the corresponding correction files",
     )
-    parser.add_argument("--isCorr", action="store_true", help="Run with SFs")
     parser.add_argument(
         "--isSyst",
         default=None,
         type=str,
         choices=[None, "all", "weight_only", "JERC_split"],
         help="Run with systematics, all, weights_only(no JERC uncertainties included),JERC_split, None",
     )
-    parser.add_argument(
-        "--isJERC", action="store_true", help="JER/JEC implemented to jet"
-    )
     parser.add_argument("--isArray", action="store_true", help="Output root files")
     parser.add_argument(
         "--noHist", action="store_true", help="Not output coffea histogram"
@@ -114,13 +110,7 @@ def get_main_parser():
         metavar="N",
         help="Number of events per process chunk",
     )
-    parser.add_argument(
-        "--retries",
-        type=int,
-        default=25,
-        metavar="N",
-        help="Number of retries for coffea processor",
-    )
+    parser.add_argument("--skipbadfiles", action="store_true", help="Skip bad files.")
     parser = get_condor_submitter_parser(parser)
     return parser