updates

Author: loubnabnl

pii/ner/pii_redaction/run_pii_slurm.py

@@ -193,14 +193,13 @@ def makejob(JOB_NAME="pii-redaction", LANG=None, TEXT_COLUMN="content"):
         "cmake",
     ]
     for i in range(args.start, args.end + 1):
-        #language = languages[i]
-        language="github-issues-filtered-structured"
+        language = languages[i]
         print(f"Submitting jobs for experiment on language {language}")
         job_name = f"{language}-pii-redaction-idx_{i}"
         job = makejob(
             JOB_NAME=job_name,
             LANG=language,
-            TEXT_COL=args.text_column,
+            TEXT_COLUMN=args.text_column,
         )
         # submit the job
         print(f"Job for lang {language} ready and saved at jobs/{job_name}.sbatch")

pii/ner/pii_redaction/utils.py

@@ -120,6 +120,7 @@ def redact_pii_text(text, secrets, replacements, add_references=False):
         step = 0
         last_text = text
         for secret in secrets:
+            # some post-processing 
             if secret["tag"] in IGNORE or not is_secret(secret["value"]):
                 continue
             if secret["tag"] == "IP_ADDRESS":