ff

.flake8

@@ -1,5 +1,5 @@
 [flake8]
-ignore = E203, E266, E501, W503
-max-line-length = 80
+ignore = E203, E266, E501, W503, B006, B007, B008, F401, C416, B950, B904
+max-line-length = 88
 max-complexity = 18
-select = B,C,E,F,W,T4,B9
+select = B,C,E,F,W,T4,B9

.github/workflows/dev-cicd.yml

@@ -0,0 +1,64 @@
+name: dev-cicd-setup-and-test
+
+on:
+  push:
+    branches:
+      - dev
+  pull_request:
+    branches:
+      - dev
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - name: Install pre-commit
+        run: pip install pre-commit
+      - name: Run pre-commit
+        run: pre-commit run --all-files
+
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Tesseract OCR
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y software-properties-common
+          sudo add-apt-repository -y ppa:alex-p/tesseract-ocr-devel
+          sudo apt-get update
+          sudo apt-get install -y tesseract-ocr libtesseract-dev
+          tesseract --version
+          dpkg -l | grep tesseract
+      - name: Verify Tesseract Installation
+        run: |
+          which tesseract
+          tesseract --list-langs
+      - name: Install Dependencies
+        run: |
+          pip install -U pip
+          pip install -e .
+          pip install tox just pre-commit
+      - name: Run Tests with tox
+        run: tox -- --cov datafog --cov-report xml --cov-report term --codeblocks
+      - name: Submit to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./coverage.xml
+          flags: unittests
+          name: codecov-umbrella

.github/workflows/feature-cicd.yml

@@ -0,0 +1,64 @@
+name: feature-cicd-setup-and-test
+
+on:
+  push:
+    branches:
+      - feature/*
+  pull_request:
+    branches:
+      - feature/*
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - name: Install pre-commit
+        run: pip install pre-commit
+      - name: Run pre-commit
+        run: pre-commit run --all-files
+
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Tesseract OCR
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y software-properties-common
+          sudo add-apt-repository -y ppa:alex-p/tesseract-ocr-devel
+          sudo apt-get update
+          sudo apt-get install -y tesseract-ocr libtesseract-dev
+          tesseract --version
+          dpkg -l | grep tesseract
+      - name: Verify Tesseract Installation
+        run: |
+          which tesseract
+          tesseract --list-langs
+      - name: Install Dependencies
+        run: |
+          pip install -U pip
+          pip install -e .
+          pip install tox just pre-commit
+      - name: Run Tests with tox
+        run: tox -- --cov datafog --cov-report xml --cov-report term --codeblocks
+      - name: Submit to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./coverage.xml
+          flags: unittests
+          name: codecov-umbrella

.github/workflows/main-cicd.yml

@@ -0,0 +1,64 @@
+name: main-cicd-setup-and-test
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - name: Install pre-commit
+        run: pip install pre-commit
+      - name: Run pre-commit
+        run: pre-commit run --all-files
+
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10"]
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Tesseract OCR
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y software-properties-common
+          sudo add-apt-repository -y ppa:alex-p/tesseract-ocr-devel
+          sudo apt-get update
+          sudo apt-get install -y tesseract-ocr libtesseract-dev
+          tesseract --version
+          dpkg -l | grep tesseract
+      - name: Verify Tesseract Installation
+        run: |
+          which tesseract
+          tesseract --list-langs
+      - name: Install Dependencies
+        run: |
+          pip install -U pip
+          pip install -e .
+          pip install tox just pre-commit
+      - name: Run Tests with tox
+        run: tox -- --cov datafog --cov-report xml --cov-report term --codeblocks
+      - name: Submit to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./coverage.xml
+          flags: unittests
+          name: codecov-umbrella

.github/workflows/publish-pypi.yml

@@ -0,0 +1,45 @@
+name: PyPI Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: "Version to release (e.g., 1.2.3)"
+        required: true
+      confirm_tests:
+        description: "Confirm all tests have passed"
+        type: boolean
+        required: true
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    if: github.event.inputs.confirm_tests == 'true'
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install build twine
+      - name: Build package
+        run: python -m build
+      - name: Create GitHub Release
+        env:
+          GITHUB_TOKEN: ${{ secrets.pypi }}
+        run: |
+          git config user.name github-actions
+          git config user.email github-actions@github.com
+          git tag v${{ github.event.inputs.version }}
+          git push origin v${{ github.event.inputs.version }}
+          gh release create v${{ github.event.inputs.version }} --generate-notes
+      - name: Publish to PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+        run: twine upload dist/*

.gitignore

@@ -9,18 +9,24 @@ build/
 *.egg-info/
 .pytest_cache/
 .tox/
-/src/datafog/__pycache__/
-/src/datafog/pii_tools/__pycache__/
-/tests/__pycache__/
-/tests/scratch.py
+src/datafog/__pycache__/
+src/datafog/pii_tools/__pycache__/
+tests/__pycache__/
+tests/scratch.py
+tests/.datafog_env/
 node_modules/
 datafog_debug.log
 sotu_2023.txt
 .DS_Store
-/venv/
+venv/
 datafog-python/datafog/processing/image_processing/__pycache__/
 datafog-python/datafog/processing/text_processing/__pycache__/
 datafog-python/datafog/services/__pycache__/
 datafog-python/datafog/processing/__pycache__/
 datafog-python/datafog/__pycache__/
-
+.env
+coverage.xml
+htmlcov/
+.venv/
+node_modules/
+.DS_Store

.pre-commit-config.yaml

@@ -4,19 +4,25 @@ repos:
     hooks:
       - id: isort
         args: [--profile=black]
+        exclude: .venv
 
   - repo: https://github.com/psf/black
     rev: 24.2.0
     hooks:
       - id: black
         language_version: python3
+        exclude: .venv
 
   - repo: https://github.com/pycqa/flake8
     rev: 7.0.0
     hooks:
       - id: flake8
+        args: [--max-line-length=88] # Match Black's line length
+        additional_dependencies: [flake8-bugbear, flake8-comprehensions]
+        exclude: .venv
 
   - repo: https://github.com/pre-commit/mirrors-prettier
     rev: v4.0.0-alpha.8
     hooks:
       - id: prettier
+        exclude: .venv