Skip to content

Commit 87b5858

Browse files
msuthar09msuthar09
authored
Merge pull request #12 from righteouslabs/ready_to_merge_to_main
Ready to merge to main
2 parents 6f3a734 + 9b66404 commit 87b5858

35 files changed

+2037
-44
lines changed

‎.github/actions/run-python-tests/action.yaml‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
name: 'Run python test cases'
2+
description: 'Run standardized Python test cases and pre-build actions'
3+
4+
inputs:
5+
python-version:
6+
description: 'Python version to use'
7+
required: true
8+
default: '3.10'
9+
10+
runs:
11+
using: "composite"
12+
13+
steps:
14+
15+
- name: Set up Python ${{ inputs.python-version }} 🐍
16+
uses: actions/setup-python@v3
17+
with:
18+
python-version: ${{ inputs.python-version }}
19+
20+
- name: Add conda to system path ➡️
21+
shell: bash
22+
run: |
23+
# $CONDA is an environment variable pointing to the root of the miniconda directory
24+
echo $CONDA/bin >> $GITHUB_PATH
25+
26+
- name: Install dependencies ⬇️
27+
shell: bash
28+
run: |
29+
# Update Conda
30+
conda update -n base -c defaults conda
31+
# Update Conda packages
32+
conda env update --file environment.yml --name base
33+
34+
- name: Run pre-commit checks ✔️
35+
uses: pre-commit/action@v3.0.0
36+
37+
- name: Lint with flake8 📐
38+
shell: bash
39+
run: |
40+
# stop the build if there are Python syntax errors or undefined names
41+
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
42+
# exit-zero treats all errors as warnings
43+
flake8 . --count --exit-zero --statistics
44+
45+
- name: Lint with Black 📐
46+
shell: bash
47+
run: |
48+
# stop the build if there are Python issues with Black
49+
python -m black --check .
50+
51+
- name: Test with pytest 🧪
52+
shell: bash
53+
run: |
54+
python -m pytest
55+
56+
- name: ☂ Upload Code Coverage
57+
uses: codecov/codecov-action@v3.1.0

‎.github/workflows/test-deploy.yml‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
2+
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
3+
4+
name: Package Tests
5+
6+
on:
7+
push:
8+
branches: [ "*" ]
9+
pull_request:
10+
branches: [ "*" ]
11+
workflow_call:
12+
release:
13+
types: [released]
14+
15+
env:
16+
PYTHON_VERSION_FOR_PUBLISH: "3.10"
17+
18+
jobs:
19+
20+
test:
21+
22+
runs-on: ubuntu-latest
23+
24+
strategy:
25+
fail-fast: false
26+
matrix:
27+
python-version: ["3.8", "3.9", "3.10"]
28+
29+
steps:
30+
- uses: actions/checkout@v3
31+
32+
- uses: ./.github/actions/run-python-tests
33+
with:
34+
python-version: ${{ matrix.python-version }}
35+
36+
deploy:
37+
38+
needs: test
39+
40+
runs-on: ubuntu-latest
41+
42+
if: ${{ github.event_name == 'release' && github.event.action == 'released' }}
43+
44+
steps:
45+
- uses: actions/checkout@v3
46+
47+
- name: ✔ Run tests
48+
uses: ./.github/actions/run-python-tests
49+
with:
50+
python-version: ${{ env.PYTHON_VERSION_FOR_PUBLISH }}
51+
52+
- name: 🚧 Build a binary wheel and a source tarball
53+
run: |
54+
python -m build --sdist --wheel --outdir dist/ .
55+
56+
- name: 📦 Publish distribution to Test.PyPI.org
57+
uses: pypa/gh-action-pypi-publish@master
58+
with:
59+
password: ${{ secrets.TEST_PYPI_API_TOKEN }}
60+
repository_url: https://test.pypi.org/legacy/
61+
62+
- name: 📦 Publish distribution to PyPI.org
63+
uses: pypa/gh-action-pypi-publish@master
64+
with:
65+
password: ${{ secrets.PYPI_API_TOKEN }}
66+
repository_url: https://upload.pypi.org/legacy/

‎.gitignore‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ htmlcov/
4646
.cache
4747
nosetests.xml
4848
coverage.xml
49+
test-output.xml
4950
*.cover
5051
*.py,cover
5152
.hypothesis/
@@ -127,3 +128,15 @@ dmypy.json
127128

128129
# Pyre type checker
129130
.pyre/
131+
132+
# Local History for Visual Studio Code
133+
.history/
134+
135+
# Local testing files
136+
testing.ipynb
137+
tests/dummy.csv
138+
tests/mockup.yaml
139+
tests/mockup2.yml
140+
tests/mockup3.yml
141+
tests/mockup4.yml
142+
tests/test.py

‎.pre-commit-config.yaml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# Usage:
2+
# pre-commit install
3+
# pre-commit run --all-files
4+
5+
# Reference: https://pre-commit.com/#usage
6+
7+
repos:
8+
9+
- repo: https://github.com/pre-commit/pre-commit-hooks
10+
rev: v2.3.0
11+
hooks:
12+
- id: check-yaml
13+
- id: end-of-file-fixer
14+
- id: trailing-whitespace
15+
16+
- repo: https://github.com/psf/black
17+
rev: 22.3.0
18+
hooks:
19+
- id: black
20+
21+
# # Run pytest runs locally
22+
# - repo: local
23+
# hooks:
24+
# - id: pytest-check
25+
# name: pytest-check
26+
# stages: [manual]
27+
# types: [python]
28+
# entry: pytest
29+
# language: system
30+
# pass_filenames: false
31+
# always_run: true

‎.vscode/.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# Ignore VS Code user preferences
2+
settings.json

‎.vscode/ReadMe.md‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
# <img src="https://code.visualstudio.com/assets/images/code-stable.png" height=24 /> Visual Studio Code Folder
2+
3+
This folder contains files needed to have a configured IDE experience with Visual Studio Code 💖
4+
5+
## Folder Contents:
6+
7+
* `extensions.json` : Contains recommend extensions for VS Code
8+
* `launch.json` : Contains debug profiles for launching Python using VS Code
9+
* `settings.json.template` : A set of recommended settings for VS Code to get best user experience
10+
* `tasks.json` : Contains automation tasks for convenience

‎.vscode/extensions.json‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
{
2+
// See https://go.microsoft.com/fwlink/?LinkId=827846 to learn about workspace recommendations.
3+
// Extension identifier format: ${publisher}.${name}. Example: vscode.csharp
4+
// List of extensions which should be recommended for users of this workspace.
5+
"recommendations": [
6+
// Python extensions
7+
"ms-python.python",
8+
"ms-python.vscode-pylance",
9+
"njpwerner.autodocstring",
10+
11+
// Development extensions
12+
"TabNine.tabnine-vscode",
13+
14+
// Utility extensions
15+
"eamodio.gitlens",
16+
"gruntfuggly.todo-tree",
17+
"IBM.output-colorizer",
18+
],
19+
// List of extensions recommended by VS Code that should not be recommended for users of this workspace.
20+
"unwantedRecommendations": []
21+
}

‎.vscode/launch.json‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
{
2+
// Use IntelliSense to learn about possible attributes.
3+
// Hover to view descriptions of existing attributes.
4+
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
5+
"version": "0.2.0",
6+
"configurations": [
7+
{
8+
"name": "Python: Current File",
9+
"type": "python",
10+
"request": "launch",
11+
"program": "${file}",
12+
"console": "integratedTerminal",
13+
"justMyCode": true,
14+
"cwd": "${workspaceFolder}",
15+
"env": {
16+
"PYTHONPATH": "${workspaceFolder}",
17+
},
18+
},
19+
{
20+
"name": "Python: Specific File",
21+
"type": "python",
22+
"request": "launch",
23+
"program": "${workspaceFolder}/test.py",
24+
"console": "integratedTerminal",
25+
"justMyCode": true,
26+
"cwd": "${workspaceFolder}",
27+
"env": {
28+
"PYTHONPATH": "${workspaceFolder}",
29+
},
30+
},
31+
{
32+
"name": "Python: Python Tests",
33+
"type": "python",
34+
"request": "launch",
35+
"module": "pytest",
36+
"console": "integratedTerminal",
37+
"justMyCode": true,
38+
"cwd": "${workspaceFolder}",
39+
"args": [
40+
// If you want to run a specific test only
41+
// "./tests/test_etl.py::TestAddArgumentImports::test_add_argument_imports"
42+
"./tests/test_etl.py::TestCreateEngineConnection::test_create_engine_connection"
43+
],
44+
"env": {
45+
"PYTHONPATH": "${workspaceFolder}",
46+
},
47+
},
48+
{
49+
"name": "Python: Yaml Test 1",
50+
"type": "python",
51+
"request": "launch",
52+
"module": "pandas_etl",
53+
"console": "integratedTerminal",
54+
"args": ["--file", "${workspaceFolder}/tests/mockup.yaml"],
55+
"justMyCode": true,
56+
"cwd": "${workspaceFolder}",
57+
"env": {
58+
"PYTHONPATH": "${workspaceFolder}",
59+
},
60+
},
61+
{
62+
"name": "Python: Yaml Test 2",
63+
"type": "python",
64+
"request": "launch",
65+
"module": "pandas_etl",
66+
"console": "integratedTerminal",
67+
"args": ["--file", "${workspaceFolder}/tests/mockup4.yml", "--var", "varName1=varValue1",],
68+
"justMyCode": true,
69+
"cwd": "${workspaceFolder}",
70+
"env": {
71+
"PYTHONPATH": "${workspaceFolder}",
72+
},
73+
},
74+
{
75+
"name": "Python: Yaml Test 3",
76+
"type": "python",
77+
"request": "launch",
78+
"module": "pandas_etl",
79+
"console": "integratedTerminal",
80+
"args": ["--file", "${workspaceFolder}/tests/etl_definition_folder/pipelines/pandas_pipeline_recovery_1.yaml"],
81+
"justMyCode": true,
82+
"cwd": "${workspaceFolder}",
83+
"env": {
84+
"PYTHONPATH": "${workspaceFolder}",
85+
},
86+
},
87+
]
88+
}

‎.vscode/settings.json.template‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{
2+
"python.linting.cwd": "${workspaceFolder}",
3+
"python.testing.pytestArgs": [
4+
"tests"
5+
],
6+
"python.formatting.provider": "black",
7+
"python.analysis.extraPaths": [
8+
"${workspaceFolder}"
9+
],
10+
"python.testing.unittestEnabled": false,
11+
"python.testing.pytestEnabled": true
12+
}

‎.vscode/tasks.json‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
{
2+
// See https://go.microsoft.com/fwlink/?LinkId=733558
3+
// for the documentation about the tasks.json format
4+
"version": "2.0.0",
5+
"tasks": [
6+
{
7+
"label": "local: conda env update",
8+
"type": "shell",
9+
"command": "conda",
10+
"detail": "Setup Anaconda environment locally",
11+
"args": [
12+
"env",
13+
"update",
14+
"--name",
15+
"pandas-etl"
16+
],
17+
"problemMatcher": [],
18+
},
19+
{
20+
"label": "local: python: black: current file",
21+
"type": "shell",
22+
"detail": "Format current python file based on black style standard",
23+
"command": "${config:python.defaultInterpreterPath}",
24+
"args": [
25+
// Call the python black module
26+
"-m",
27+
"black",
28+
// Specify the file that needs to be formatted
29+
"${file}",
30+
],
31+
"problemMatcher": [],
32+
},
33+
{
34+
"label": "local: create dependency documentation folder",
35+
"type": "shell",
36+
"detail": "Used as a common task to create subfolders needed for output",
37+
"command": "mkdir",
38+
"windows": {
39+
"args": [
40+
"-f",
41+
"${workspaceFolder}/docs/public/_static",
42+
],
43+
},
44+
"linux": {
45+
"args": [
46+
"-p",
47+
"${workspaceFolder}/docs/public/_static",
48+
],
49+
},
50+
"problemMatcher": [],
51+
},
52+
{
53+
"label": "python: pytest: run all tests",
54+
"type": "process",
55+
"detail": "Run all test cases for this project (also generates HTML coverage report)",
56+
"command": "${config:python.defaultInterpreterPath}",
57+
"args": [
58+
// Call the python pytest module
59+
"-m",
60+
"pytest",
61+
// Pytest command line arguments
62+
"-c", "${workspaceFolder}/tests/pytest.ini",
63+
"${workspaceFolder}",
64+
],
65+
"problemMatcher": [],
66+
"group": {
67+
"kind": "test",
68+
"isDefault": true
69+
},
70+
},
71+
],
72+
}

0 commit comments

Comments
 (0)