feat: add validation workflow from MC & add 2023 files

.github/workflows/BTA_workflow.yml

@@ -87,6 +87,7 @@ jobs:
         chmod 400 $HOME/.globus/userkey.pem
         openssl rand -out $HOME/.rnd  -hex 256
         printf "${{secrets.GRID_PASSWORD}}" | voms-proxy-init --voms cms --vomses ${X509_VOMSES} --debug --pwstdin
+        chmod 755 /usr/share/miniconda3/envs/btv_coffea/etc/grid-security/certificates
 
     - name: Test xrootd
       run: |

.github/workflows/ctag_DY_workflow.yml

@@ -86,6 +86,7 @@ jobs:
         chmod 400 $HOME/.globus/userkey.pem
         openssl rand -out $HOME/.rnd  -hex 256
         printf "${{secrets.GRID_PASSWORD}}" | voms-proxy-init --voms cms --vomses ${X509_VOMSES} --debug --pwstdin
+        chmod 755 /usr/share/miniconda3/envs/btv_coffea/etc/grid-security/certificates
 
     - name: Test xrootd
       run: |

.github/workflows/ctag_Wc_workflow.yml

@@ -90,7 +90,7 @@ jobs:
         chmod 400 $HOME/.globus/userkey.pem
         openssl rand -out $HOME/.rnd  -hex 256
         printf "${{secrets.GRID_PASSWORD}}" | voms-proxy-init --voms cms --vomses ${X509_VOMSES} --debug --pwstdin
-
+        chmod 755 /usr/share/miniconda3/envs/btv_coffea/etc/grid-security/certificates
     - name: Test xrootd
       run: |
         xrdcp root://eoscms.cern.ch//eos/cms/store/group/phys_btag/nano-commissioning/test_w_dj.root .

.github/workflows/ctag_ttbar_workflow.yml

@@ -87,6 +87,7 @@ jobs:
         chmod 400 $HOME/.globus/userkey.pem
         openssl rand -out $HOME/.rnd  -hex 256
         printf "${{secrets.GRID_PASSWORD}}" | voms-proxy-init --voms cms --vomses ${X509_VOMSES} --debug --pwstdin
+        chmod 755 /usr/share/miniconda3/envs/btv_coffea/etc/grid-security/certificates
 
     - name: Test xrootd
       run: |

.github/workflows/ttbar_workflow.yml

@@ -87,7 +87,7 @@ jobs:
         chmod 400 $HOME/.globus/userkey.pem
         openssl rand -out $HOME/.rnd  -hex 256
         printf "${{secrets.GRID_PASSWORD}}" | voms-proxy-init --voms cms --vomses ${X509_VOMSES} --debug --pwstdin
-
+        chmod 755 /usr/share/miniconda3/envs/btv_coffea/etc/grid-security/certificates
     - name: Test xrootd
       run: |
         xrdcp root://eoscms.cern.ch//eos/cms/store/group/phys_btag/nano-commissioning/test_w_dj.root .

README.md

@@ -239,6 +239,10 @@ However, some sites have certain restrictions for various reasons, in particular
 
 
 Memory usage is also useful to adapt to cluster. Check the memory by calling  `memory_usage_psutil()` from `helpers.func.memory_usage_psutil` to optimize job size. Example with `ectag_Wc_sf` summarized below.
+
+<details><summary>details</summary>
+<p>
+
  Type        |Array+Hist |  Hist only| Array Only|
 | :---:   | :---: | :---: | :---: |
 DoubleMuon (BTA,BTV_Comm_v2)| 1243MB |	848MB	|1249MB|
@@ -248,6 +252,8 @@ WJets_inc (BTA,BTV_Comm_v2)| 1243MB	|848MB	|1249MB|
 WJets_inc (PFCands, BTV_Comm_v1)|1650MB	|1274MB	|1632MB
 WJets_inc (Nano_v11)|1183MB	|630MB	|1180MB|
 
+</p>
+</details>
 
 ### Sites configuration with dask/parsl schedular
 
@@ -318,6 +324,8 @@ This utility is currently adapted for the lxplus and cmsconnect condor systems.
 
 After executing the command, a new folder will be created, preparing the submission. Follow the on-screen instructions and utilize `condor_submit ...` to submit the jdl file. The output will be transferred to the designated XRootD destination.
 
+The script provided by Pablo to resubmit failure jobs in `script/missingFiles.py` from the original job folder.
+
 <details><summary>Frequent issues for standalone condor jobs submission
 </summary>
 <p>
@@ -333,6 +341,7 @@ After executing the command, a new folder will be created, preparing the submiss
 </details>
 
 
+
 ## Make the dataset json files
 
 Use `fetch.py` in folder `scripts/` to obtain your samples json files. You can create `$input_list` ,which can be a list of datasets taken from CMS DAS , and create the json contains `dataset_name:[filelist]`. One can specify the local path in that input list for samples not published in CMS DAS.

condor/execute.sh

@@ -62,7 +62,10 @@ OPTS="--wf ${ARGS[workflow]} --year ${ARGS[year]} --campaign ${ARGS[campaign]} -
 if [ "${ARGS[voms]}" != "null" ]; then
     OPTS="$OPTS --voms ${ARGS[voms]}"
 fi
-for key in  isSyst isArray noHist overwrite skipbadfiles; do
+if [ "${ARGS[isSyst]}" != "null" ]; then
+    OPTS="$OPTS --isSyst ${ARGS[isSyst]}"
+fi
+for key in  isArray noHist overwrite skipbadfiles; do
     if [ "${ARGS[$key]}" == true ]; then
         OPTS="$OPTS --$key"
     fi

runner.py

@@ -80,6 +80,8 @@ def get_main_parser():
             "Winter22Run3",
             "Summer22Run3",
             "Summer22EERun3",
+            "Summer23",
+            "Summer23BPix",
             "2018_UL",
             "2017_UL",
             "2016preVFP_UL",
@@ -89,9 +91,9 @@ def get_main_parser():
     )
     parser.add_argument(
         "--isSyst",
-        default=None,
+        default=False,
         type=str,
-        choices=[None, "all", "weight_only", "JERC_split"],
+        choices=[False, "all", "weight_only", "JERC_split"],
         help="Run with systematics, all, weights_only(no JERC uncertainties included),JERC_split, None",
     )
     parser.add_argument("--isArray", action="store_true", help="Output root files")
@@ -227,7 +229,7 @@ def get_main_parser():
     ogoutput = args.output
     histoutdir = ogoutput.split(".")[0]
     coffeaoutput = f"{histoutdir}/{ogoutput}"
-    outdir = histoutdir
+    outdir = "arrays_" + histoutdir
     basename = ogoutput.replace(".coffea", "").replace("hists_", "")
     if args.output == parser.get_default("output"):
         index = args.samplejson.rfind("/") + 1
@@ -275,7 +277,7 @@ def get_main_parser():
             _new_dict = {}
             print("Will only proces the following datasets:")
             for k, v in sample_dict.items():
-                if k.lstrip("/").startswith(args.only.rstrip("*")):
+                if args.only.replace("*", "") in k:
                     print("    ", k)
                     _new_dict[k] = v
             sample_dict = _new_dict

scripts/comparison.py

@@ -60,8 +60,8 @@
 parser.add_argument(
     "--flow",
     type=str,
-    default="show",
-    help="str, optional {None, 'show', 'sum'} Whether plot the under/overflow bin. If 'show', add additional under/overflow bin. If 'sum', add the under/overflow bin content to first/last bin.",
+    default="none",
+    help="str, optional {none, 'show', 'sum'} Whether plot the under/overflow bin. If 'show', add additional under/overflow bin. If 'sum', add the under/overflow bin content to first/last bin.",
 )
 parser.add_argument("--ext", type=str, default="", help="prefix name/btv name tag")
 parser.add_argument("--com", default="13.6", type=str, help="sqrt(s) in TeV")
@@ -188,21 +188,31 @@
 else:
     var_set = args.variable.split(",")
 for index, discr in enumerate(var_set):
+    if not isinstance(collated[args.ref][discr], hist.hist.Hist):
+        continue
     allaxis = {}
-
     if (
         discr not in collated[args.ref].keys()
         or (collated[args.ref][discr].values() == 0).all()
     ):
         print(discr, "not in file or empty")
-    for c in args.compared.split(","):
-        if discr not in collated[c].keys() or (collated[c][discr].values() == 0).all():
-            print(discr, "not in file or empty")
+        continue
+    checkempty = [
+        False
+        for c in args.compared.split(",")
+        if discr not in collated[c].keys() or (collated[c][discr].values() == 0).all()
+    ]
+    if all(checkempty) == False:
+        print(discr, "not in file or empty")
+        continue
+
         continue
     if "flav" in collated[args.ref][discr].axes.name:
         allaxis["flav"] = sum
     if "syst" in collated[args.ref][discr].axes.name:
         allaxis["syst"] = "nominal"
+        if "btag" in discr:
+            allaxis["syst"] = "noSF"
     if "osss" in collated[args.ref][discr].axes.name:  ## do dominal OS-SS
         if args.splitOSSS is None:  # OS-SS
             collated[args.ref][discr] = (
@@ -225,12 +235,13 @@
         else:
             rebin = np.array([float(i) for i in args.autorebin.split(",")])
             do_xerr = True
-        collated["mc"][discr] = rebin_hist(
-            collated["mc"][discr], collated["mc"][discr].axes[-1].name, rebin
-        )
-        collated["data"][discr] = rebin_hist(
-            collated["data"][discr], collated["data"][discr].axes[-1].name, rebin
+        collated[args.ref][discr] = rebin_hist(
+            collated[args.ref][discr], collated[args.ref][discr].axes[-1].name, rebin
         )
+        for c in args.compared.split(","):
+            collated[c][discr] = rebin_hist(
+                collated[c][discr], collated[c][discr].axes[-1].name, rebin
+            )
 
     ## FIXME: Set temporary fix for the x-axis
     if args.xlabel is not None:
@@ -429,12 +440,16 @@
                 xerr=do_xerr,
                 flow=args.flow,
             )
+            if len(ax.get_lines()) > 2 * len(args.compared.split(",")):
+                colors = ax.get_lines()[(i + 1) * 2]
+            else:
+                colors = ax.get_lines()[i + 1]
             plotratio(
                 collated[c][discr][allaxis],
                 collated[args.ref][discr][allaxis],
                 ax=rax,
                 denom_fill_opts=None,
-                error_opts={"color": ax.get_lines()[i + 1].get_color()},
+                error_opts={"color": colors.get_color()},
                 clear=False,
                 xerr=do_xerr,
                 flow=args.flow,
@@ -452,9 +467,10 @@
         ax.set_ylabel(args.ylabel)
         rax.set_ylabel("Other/Ref")
         ax.legend(loc=1)
-        rax.set_ylim(0.0, 2.0)
+        rax.axhline(1.0, ls=":")
+        rax.set_ylim(0.5, 1.5)
 
-        at = AnchoredText(input_txt + "\n" + args.ext, loc=2, frameon=False)
+        at = AnchoredText(input_txt + "\n" + text, loc=2, frameon=False)
         ax.add_artist(at)
         hep.mpl_magic(ax=ax)
         ax.set_ylim(bottom=0)