Scrubbing

.gitignore

@@ -21,4 +21,12 @@ cache
 performance
 
 # Generated when adding editable dependencies in requirements.txt (-e)
-src
+src
+# VSCode
+*.vscode
+
+# venv Folder
+venv/*
+
+# HTML
+*.html

README.md

@@ -31,6 +31,7 @@ source .venv/bin/activate
 pip install wheel
 pip install -r requirements.txt
 pip install -e .
+python -m spacy download en_core_web_trf
 alembic upgrade head
 pytest
 ```
@@ -209,4 +210,4 @@ Please submit any issues to https://github.com/MLDSAI/openadapt/issues with the
 following information:
 
 - Problem description (please include any relevant console output and/or screenshots)
-- Steps to reproduce (please help others to help you!)
+- Steps to reproduce (please help others to help you!)

alembic.ini

@@ -102,4 +102,4 @@ formatter = generic
 
 [formatter_generic]
 format = %(levelname)-5.5s [%(name)s] %(message)s
-datefmt = %H:%M:%S
+datefmt = %H:%M:%S

alembic/env.py

@@ -89,4 +89,4 @@ def run_migrations_online() -> None:
 if context.is_offline_mode():
     run_migrations_offline()
 else:
-    run_migrations_online()
+    run_migrations_online()

alembic/script.py.mako

@@ -21,4 +21,4 @@ def upgrade() -> None:
 
 
 def downgrade() -> None:
-    ${downgrades if downgrades else "pass"}
+    ${downgrades if downgrades else "pass"}

openadapt/config.py

@@ -4,6 +4,10 @@
 
 from dotenv import load_dotenv
 from loguru import logger
+from presidio_analyzer import AnalyzerEngine
+from presidio_analyzer.nlp_engine import NlpEngineProvider
+from presidio_anonymizer import AnonymizerEngine
+from presidio_image_redactor import ImageRedactorEngine, ImageAnalyzerEngine
 
 
 _DEFAULTS = {
@@ -43,3 +47,56 @@ def getenv_fallback(var_name):
     for key, val in locals().items():
         if not key.startswith("_") and key.isupper():
             logger.info(f"{key}={val}")
+
+
+# SCRUBBING CONFIGURATIONS
+
+# SCRUB_CONFIG = {
+#     "nlp_engine_name": "spacy",
+#     "models": [{"lang_code": "en", "model_name": "en_core_web_lg"}],
+# }
+# SCRUB_PROVIDER = NlpEngineProvider(nlp_configuration=SCRUB_CONFIG)
+# NLP_ENGINE = SCRUB_PROVIDER.create_engine()
+ANALYZER = AnalyzerEngine(
+    # nlp_engine=NLP_ENGINE,
+    supported_languages=["en"]
+)
+ANONYMIZER = AnonymizerEngine()
+IMAGE_REDACTOR = ImageRedactorEngine(ImageAnalyzerEngine(ANALYZER))
+SCRUB_IGNORE_ENTITIES = [
+    # 'US_PASSPORT',
+    # 'US_DRIVER_LICENSE',
+    # 'CRYPTO',
+    # 'UK_NHS',
+    # 'PERSON',
+    # 'CREDIT_CARD',
+    # 'US_BANK_NUMBER',
+    # 'PHONE_NUMBER',
+    # 'US_ITIN',
+    # 'AU_ABN',
+    'DATE_TIME',
+    # 'NRP',
+    # 'SG_NRIC_FIN',
+    # 'AU_ACN',
+    # 'IP_ADDRESS',
+    # 'EMAIL_ADDRESS',
+    'URL',
+    # 'IBAN_CODE',
+    # 'AU_TFN',
+    # 'LOCATION',
+    # 'AU_MEDICARE',
+    # 'US_SSN',
+    # 'MEDICAL_LICENSE'
+]
+SCRUBBING_ENTITIES = [
+    entity
+    for entity in ANALYZER.get_supported_entities()
+    if entity not in SCRUB_IGNORE_ENTITIES
+]
+SCRUB_KEYS_HTML = [
+    'text',
+    'canonical_text',
+    'title',
+    'state'
+]
+DEFAULT_SCRUB_FILL_COLOR = (255,0,0)

openadapt/models.py

@@ -6,7 +6,7 @@
 import numpy as np
 import sqlalchemy as sa
 
-from openadapt import db, utils, window
+from openadapt import db, utils, window, scrub
 
 
 # https://groups.google.com/g/sqlalchemy/c/wlr7sShU6-k
@@ -61,6 +61,9 @@ def processed_action_events(self):
 
 class ActionEvent(db.Base):
     __tablename__ = "action_event"
+    _text_sep = "-"
+    _text_name_prefix = "<"
+    _text_name_suffix = ">"
 
     id = sa.Column(sa.Integer, primary_key=True)
     name = sa.Column(sa.String)
@@ -161,7 +164,9 @@ def _text(self, canonical=False):
                 )
             else:
                 text = key_attr
-        return text
+
+        scrubbed_text = scrub.scrub_text(text, is_hyphenated=True)
+        return scrubbed_text
 
     @property
     def text(self):
@@ -201,9 +206,6 @@ def __str__(self):
         rval = " ".join(attrs)
         return rval
 
-    _text_sep = "-"
-    _text_name_prefix = "<"
-    _text_name_suffix = ">"
 
     @classmethod
     def from_children(cls, children_dicts):

openadapt/record.py

@@ -22,7 +22,7 @@
 import fire
 import mss.tools
 
-from openadapt import config, crud, utils, window
+from openadapt import config, crud, utils, window, scrub
 
 
 EVENT_TYPES = ("screen", "action", "window")
@@ -162,6 +162,7 @@ def write_screen_event(
     assert event.type == "screen", event
     screenshot = event.data
     png_data = mss.tools.to_png(screenshot.rgb, screenshot.size)
+    # event_data = {"png_data": scrub.scrub_png_data(png_data)}
     event_data = {"png_data": png_data}
     crud.insert_screenshot(recording_timestamp, event.timestamp, event_data)
     perf_q.put((event.type, event.timestamp, utils.get_timestamp()))
@@ -182,6 +183,7 @@ def write_window_event(
     """
 
     assert event.type == "window", event
+    # scrubbed_window_data = scrub.scrub_dict(event.data)
     crud.insert_window_event(recording_timestamp, event.timestamp, event.data)
     perf_q.put((event.type, event.timestamp, utils.get_timestamp()))
 
@@ -352,6 +354,8 @@ def read_screen_events(
         if screenshot is None:
             logger.warning("screenshot was None")
             continue
+    # Scrubbing a ScreenShot
+        # scrubbed_screenshot = scrub.scrub_screenshot(screenshot)
         event_q.put(Event(utils.get_timestamp(), "screen", screenshot))
     logger.info("done")
 
@@ -522,12 +526,14 @@ def record(
     Args:
         task_description: a text description of the task that will be recorded
     """
+
+    scrubbed_task_description = scrub.scrub_text(task_description)
 
     utils.configure_logging(logger, LOG_LEVEL)
 
-    logger.info(f"{task_description=}")
+    logger.info(f"{scrubbed_task_description=}")
 
-    recording = create_recording(task_description)
+    recording = create_recording(scrubbed_task_description)
     recording_timestamp = recording.timestamp
 
     event_q = queue.Queue()