updated pre-commit, all passed locally, lazy import spark in pyspark_udfs

Author: Sid Mohan

.github/workflows/feature-cicd.yml

@@ -17,7 +17,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: "3.10"
       - name: Install pre-commit
         run: pip install pre-commit
       - name: Run pre-commit
@@ -29,8 +29,10 @@ jobs:
       matrix:
         python-version: ["3.10"]
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - name: Check out repo
+        uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install Tesseract OCR
@@ -48,15 +50,12 @@ jobs:
           tesseract --list-langs
       - name: Install Dependencies
         run: |
-          python -m venv .venv
-          source .venv/bin/activate
           pip install -U pip
           pip install -r requirements.txt
+          pip install -r requirements-dev.txt
           pip install tox just pre-commit
       - name: Run Tests with tox
-        run: |
-          source .venv/bin/activate
-          tox -- --cov datafog --cov-report xml --cov-report term --codeblocks
+        run: tox -- --cov datafog --cov-report xml --cov-report term --codeblocks
       - name: Submit to Codecov
         uses: codecov/codecov-action@v3
         with:

.pre-commit-config.yaml

@@ -4,21 +4,25 @@ repos:
     hooks:
       - id: isort
         args: [--profile=black]
+        exclude: .venv
 
   - repo: https://github.com/psf/black
     rev: 24.2.0
     hooks:
       - id: black
         language_version: python3
+        exclude: .venv
 
   - repo: https://github.com/pycqa/flake8
     rev: 7.0.0
     hooks:
       - id: flake8
         args: [--max-line-length=88] # Match Black's line length
         additional_dependencies: [flake8-bugbear, flake8-comprehensions]
+        exclude: .venv
 
   - repo: https://github.com/pre-commit/mirrors-prettier
     rev: v4.0.0-alpha.8
     hooks:
       - id: prettier
+        exclude: .venv

.venv/bin/black

@@ -2,7 +2,9 @@
 # -*- coding: utf-8 -*-
 import re
 import sys
+
 from black import patched_main
-if __name__ == '__main__':
-    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
     sys.exit(patched_main())

.venv/bin/blackd

@@ -2,7 +2,9 @@
 # -*- coding: utf-8 -*-
 import re
 import sys
+
 from blackd import patched_main
-if __name__ == '__main__':
-    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
     sys.exit(patched_main())

.venv/bin/blacken-docs

@@ -2,7 +2,9 @@
 # -*- coding: utf-8 -*-
 import re
 import sys
+
 from blacken_docs import main
-if __name__ == '__main__':
-    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
     sys.exit(main())

.venv/bin/chardetect

@@ -2,7 +2,9 @@
 # -*- coding: utf-8 -*-
 import re
 import sys
+
 from chardet.cli.chardetect import main
-if __name__ == '__main__':
-    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
     sys.exit(main())

.venv/bin/coverage

@@ -2,7 +2,9 @@
 # -*- coding: utf-8 -*-
 import re
 import sys
+
 from coverage.cmdline import main
-if __name__ == '__main__':
-    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
     sys.exit(main())

.venv/bin/coverage-3.10

@@ -2,7 +2,9 @@
 # -*- coding: utf-8 -*-
 import re
 import sys
+
 from coverage.cmdline import main
-if __name__ == '__main__':
-    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
     sys.exit(main())

.venv/bin/coverage3

@@ -2,7 +2,9 @@
 # -*- coding: utf-8 -*-
 import re
 import sys
+
 from coverage.cmdline import main
-if __name__ == '__main__':
-    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
     sys.exit(main())

.venv/bin/docutils

@@ -2,7 +2,9 @@
 # -*- coding: utf-8 -*-
 import re
 import sys
+
 from docutils.__main__ import main
-if __name__ == '__main__':
-    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
     sys.exit(main())

.venv/bin/f2py

@@ -2,7 +2,9 @@
 # -*- coding: utf-8 -*-
 import re
 import sys
+
 from numpy.f2py.f2py2e import main
-if __name__ == '__main__':
-    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
     sys.exit(main())

.venv/bin/flake8

@@ -2,7 +2,9 @@
 # -*- coding: utf-8 -*-
 import re
 import sys
+
 from flake8.main.cli import main
-if __name__ == '__main__':
-    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
     sys.exit(main())

.venv/bin/isort

@@ -2,7 +2,9 @@
 # -*- coding: utf-8 -*-
 import re
 import sys
+
 from isort.main import main
-if __name__ == '__main__':
-    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
     sys.exit(main())

.venv/bin/isort-identify-imports

@@ -2,7 +2,9 @@
 # -*- coding: utf-8 -*-
 import re
 import sys
+
 from isort.main import identify_imports_main
-if __name__ == '__main__':
-    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
     sys.exit(identify_imports_main())

.venv/bin/keyring

@@ -2,7 +2,9 @@
 # -*- coding: utf-8 -*-
 import re
 import sys
+
 from keyring.cli import main
-if __name__ == '__main__':
-    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
     sys.exit(main())

.venv/bin/markdown-it

@@ -2,7 +2,9 @@
 # -*- coding: utf-8 -*-
 import re
 import sys
+
 from markdown_it.cli.parse import main
-if __name__ == '__main__':
-    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
     sys.exit(main())

.venv/bin/normalizer

@@ -2,7 +2,9 @@
 # -*- coding: utf-8 -*-
 import re
 import sys
+
 from charset_normalizer.cli import cli_detect
-if __name__ == '__main__':
-    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
     sys.exit(cli_detect())

.venv/bin/pathy

@@ -2,7 +2,9 @@
 # -*- coding: utf-8 -*-
 import re
 import sys
+
 from pathy.cli import app
-if __name__ == '__main__':
-    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
     sys.exit(app())

.venv/bin/pip

@@ -2,7 +2,9 @@
 # -*- coding: utf-8 -*-
 import re
 import sys
+
 from pip._internal.cli.main import main
-if __name__ == '__main__':
-    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
     sys.exit(main())

.venv/bin/pip3

@@ -2,7 +2,9 @@
 # -*- coding: utf-8 -*-
 import re
 import sys
+
 from pip._internal.cli.main import main
-if __name__ == '__main__':
-    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
     sys.exit(main())

.venv/bin/pip3.10

@@ -2,7 +2,9 @@
 # -*- coding: utf-8 -*-
 import re
 import sys
+
 from pip._internal.cli.main import main
-if __name__ == '__main__':
-    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
     sys.exit(main())

.venv/bin/pkginfo

@@ -2,7 +2,9 @@
 # -*- coding: utf-8 -*-
 import re
 import sys
+
 from pkginfo.commandline import main
-if __name__ == '__main__':
-    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
     sys.exit(main())

.venv/bin/py.test

@@ -2,7 +2,9 @@
 # -*- coding: utf-8 -*-
 import re
 import sys
+
 from pytest import console_main
-if __name__ == '__main__':
-    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
     sys.exit(console_main())

.venv/bin/pycodestyle

@@ -2,7 +2,9 @@
 # -*- coding: utf-8 -*-
 import re
 import sys
+
 from pycodestyle import _main
-if __name__ == '__main__':
-    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
     sys.exit(_main())

.venv/bin/pyflakes

@@ -2,7 +2,9 @@
 # -*- coding: utf-8 -*-
 import re
 import sys
+
 from pyflakes.api import main
-if __name__ == '__main__':
-    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
     sys.exit(main())

.venv/bin/pygmentize

@@ -2,7 +2,9 @@
 # -*- coding: utf-8 -*-
 import re
 import sys
+
 from pygments.cmdline import main
-if __name__ == '__main__':
-    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
     sys.exit(main())

.venv/bin/pyproject-build

@@ -2,7 +2,9 @@
 # -*- coding: utf-8 -*-
 import re
 import sys
+
 from build.__main__ import entrypoint
-if __name__ == '__main__':
-    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
     sys.exit(entrypoint())

.venv/bin/pytesseract

@@ -2,7 +2,9 @@
 # -*- coding: utf-8 -*-
 import re
 import sys
+
 from pytesseract.pytesseract import main
-if __name__ == '__main__':
-    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
     sys.exit(main())

.venv/bin/pytest

@@ -2,7 +2,9 @@
 # -*- coding: utf-8 -*-
 import re
 import sys
+
 from pytest import console_main
-if __name__ == '__main__':
-    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
+
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0])
     sys.exit(console_main())