introduce automatic pylint to CI (Netflix#316)

* introduce automatic pylint to CI

* Tox is the way

Author: jackie-ob

.github/workflows/test.yml

@@ -26,7 +26,27 @@ jobs:
           python -m pip install pycodestyle
       - name: Run Python PEP8 code style checks
         run: pycodestyle
-  
+
+  pylint:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        python-version: [3.7]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Python ${{ matrix.python-version }} dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install tox pylint
+      - name: Run Tox (pylint)
+        run: tox -e pylint
+
   unit:
     runs-on: ubuntu-latest
 

services/ui_backend_service/data/cache/client/cache_server.py

@@ -299,4 +299,6 @@ def cli(root=None,
 
 
 if __name__ == '__main__':
+    # Click magic
+    # pylint: disable=unexpected-keyword-arg
     cli(auto_envvar_prefix='MFCACHE')

services/ui_backend_service/data/cache/store.py

@@ -314,6 +314,8 @@ async def preload_dag(self, flow_name: str, run_number: str):
         """
         logger.debug("  - Preload DAG for {}/{}".format(flow_name, run_number))
         # Check first if a DAG can be generated for the run.
+        # pylint-initial-ignore: Not sure where db comes from
+        # pylint: disable=no-member
         db_response = await get_run_dag_data(self.db, flow_name, run_number)
 
         if not db_response.response_code == 200:
@@ -322,6 +324,8 @@ async def preload_dag(self, flow_name: str, run_number: str):
         # Prefer run_id over run_number
         run_id = db_response.body.get('run_id') or db_response.body['run_number']
 
+        # pylint-initial-ignore: Not sure where GenerateDag comes from
+        # pylint: disable=no-member
         res = await self.cache.GenerateDag(flow_name, run_id)
         async for event in res.stream():
             if event["type"] == "error":

services/ui_backend_service/data/db/tables/base.py

@@ -230,6 +230,8 @@ async def execute_sql(self, select_sql: str, values=[], fetch_single=False,
                 records = await cur.fetchall()
                 if serialize:
                     for record in records:
+                        # pylint-initial-ignore: Lack of __init__ makes this too hard for pylint
+                        # pylint: disable=not-callable
                         row = self._row_type(**record)
                         rows.append(row.serialize(expanded))
                 else:

services/ui_backend_service/tests/integration_tests/log_test.py

@@ -69,14 +69,14 @@ async def read_and_output(cache_client, task, logtype, limit=0, page=1, reverse_
         _, data = await _test_list_resources(cli, db, "/flows/{flow_id}/runs/{run_number}/steps/{step_name}/tasks/{task_id}/logs/out?_limit=3".format(**_task), 200, None)
         assert data == []
 
-    async def read_and_output(cache_client, task, logtype, limit=0, page=1, reverse_order=False, output_raw=False):
+    async def read_and_output_2(cache_client, task, logtype, limit=0, page=1, reverse_order=False, output_raw=False):
         assert limit == 3
         assert page == 4
         assert reverse_order is True
         assert output_raw is False
         return [], 1
 
-    with mock.patch("services.ui_backend_service.api.log.read_and_output", new=read_and_output):
+    with mock.patch("services.ui_backend_service.api.log.read_and_output", new=read_and_output_2):
         # ordering by row should be possible in reverse. should obey limit.
         _, data = await _test_list_resources(cli, db, "/flows/{flow_id}/runs/{run_number}/steps/{step_name}/tasks/{task_id}/logs/out?_order=-row&_limit=3&_page=4".format(**_task), 200, None)
         assert data == []

services/ui_backend_service/tests/unit_tests/cache_utils_test.py

@@ -78,7 +78,7 @@ def _raised(output):
         assert str(ex) == "Custom exception"
 
 
-def test_streamed_errors_exception_output():
+def test_streamed_errors_exception_output_no_re_raise():
     # should not raise any exception with re_raise set to false.
     def _re_raise(output):
         pass

tox.ini

@@ -9,6 +9,9 @@ commands = pytest --cov=services
 passenv = MF_METADATA_DB_HOST MF_METADATA_DB_PORT MF_METADATA_DB_USER MF_METADATA_DB_PSWD MF_METADATA_DB_NAME MF_UI_METADATA_PORT MF_UI_METADATA_HOST
 extras = tests
 
+[testenv:pylint]
+commands = pylint -E services --ignored-modules=psycopg2.errors,pygit2
+
 [testenv:unit]
 commands = pytest --cov=services -m unit_tests