6641-bring records till s_8 (#7)

* issue fix by limiting only bringing records till s8

* updated import by isort

* ignore needing to add local package to requirements.txt

* staticaly type the code

* adding autopep8

* allow autopep8 goinside directory

* formatting/comment error for setup.cfg file

* add flake8-isort in requirements.txt

* use isort recursive

* mypy bug --no-warn-no-return github.com/python/mypy/issues/8823

.gitignore

@@ -2,4 +2,6 @@
 .direnv
 .envrc
 __pycache__
-.mypy_cache
+.mypy_cache
+venv
+out/

Makefile

@@ -4,12 +4,27 @@ FLAKE8=flake8
 
 SRCS=ref_code_gen.py ds_status_sync.py sds_flat.py
 
-check: doctest lint static
+check: autopep autoblack lint doctest lint static
+
+autopep:
+	# "converting python code to PEP formate"
+	autopep8 .
+
+autoblack:
+	# "converting python code to BLACK formate"
+	#black .
 
 static:
-	$(MYPY) --strict $(SRCS)
+	# ds_status_sync.py:311: error: unused "type: ignore" comment
+	# ds_status_sync.py:343: error: unused "type: ignore" comment
+	# using --no-warn-no-return to hide above error
+	$(MYPY) --strict $(SRCS) --no-warn-no-return
 
-lint: $(SRCS)
+lint-import:
+	# this does import sorting for you, where flak8-isort will check it for you
+	isort .
+
+lint: $(SRCS) lint-import
 	$(FLAKE8) $(SRCS)
 
 doctest:
@@ -38,3 +53,25 @@ debug: lint check
 
 install-dev-tools:
 	pip install mypy flake8
+
+ds_sync: clean venv
+	. venv/bin/activate && \
+	which python &&  python --version &&\
+	python ds_status_sync.py --get-status REDCAP_API_TOKEN DS_KEY >out/ds_status.json \
+	python ds_status_sync.py --send-consent REDCAP_API_TOKEN DS_KEY
+
+clean:
+	rm -rf out; mkdir -p out
+
+venv: venv_clear
+	# "creating python virtual env"
+	python -m venv venv
+	. ./venv/bin/activate && \
+	pip install --upgrade pip  && \
+	pip install -r requirements.txt  && \
+	pip freeze >  requirements_pip_freeze.txt  && \
+	which pip && which python && python --version
+
+venv_clear:
+	# "deleting python virtual env"
+	rm -rf venv || true

ds_status_sync.py

@@ -22,9 +22,10 @@
 from pathlib import Path as Path_T
 from urllib.error import HTTPError
 
-from requests import Request, Session as Session_T
+from requests import Request
+from requests import Session as Session_T
 
-from sds_flat import flatten, complete
+from sds_flat import complete, flatten
 
 log = logging.getLogger(__name__)
 
@@ -36,6 +37,50 @@
 WebBuilder = py.Callable[..., Session_T]
 Record_T = py.Dict[str, str]
 
+selected_survey_key = ["record_id",
+                       "registerdate",
+                       "lastvisitdate",
+                       "lastupdatedate",
+                       "s_1_title",
+                       "s_1_completed",
+                       "s_1_played_time",
+                       "s_2_title",
+                       "s_2_completed",
+                       "s_2_played_time",
+                       "s_3_title",
+                       "s_3_completed",
+                       "s_3_played_time",
+                       "s_4_title",
+                       "s_4_completed",
+                       "s_4_played_time",
+                       "s_5_title",
+                       "s_5_completed",
+                       "s_5_played_time",
+                       "s_6_title",
+                       "s_6_completed",
+                       "s_6_played_time",
+                       "s_7_title",
+                       "s_7_completed",
+                       "s_7_played_time",
+                       "s_8_title",
+                       "s_8_completed",
+                       "s_8_played_time",
+                       "ds_connect_status_complete"]
+
+
+def select_recrods_surveys(records: py.List[Record_T],
+                           selected_survey_key: py.List[str]) \
+        -> py.List[Record_T]:
+    output_records = []
+
+    for record in records:
+        output_record = {}
+        for key in record:
+            if key in selected_survey_key:
+                output_record[key] = record[key]
+        output_records.append(output_record)
+    return output_records
+
 
 def main(argv: py.List[str], env: py.Dict[str, str], stdout: py.IO[str],
          cwd: Path_T, now: py.Callable[[], datetime],
@@ -55,6 +100,7 @@ def study(api_passkey: str) -> DSConnectStudy:
         rc = REDCapAPI(REDCAP_API, make_session(), env[api_passkey])
         status = ds.getstatus([DS_DETERMINED])
         records = list(complete(STATUS_FORM)(flatten(status)))
+        records = select_recrods_surveys(records, selected_survey_key)
         json.dump({'status': status, 'records': records}, stdout, indent=2)
         rc.import_records(records)
     elif '--send-consent' in argv:
@@ -368,7 +414,7 @@ def _script_io() -> None:
         from datetime import datetime
         from os import environ
         from pathlib import Path
-        from sys import argv, stdout, stderr
+        from sys import argv, stderr, stdout
 
         from requests import Session
 

ref_code_gen.py

@@ -21,14 +21,14 @@
      {'record_id': 'SC-_TEST_00014', 'redcap_data_access_group': 'sc'}]
 """
 
+import json
+import logging
+import typing as py
 from binascii import crc32
 from pprint import pformat
 from urllib.error import HTTPError
 from urllib.parse import urlencode
 from urllib.request import OpenerDirector as OpenerDirector_T
-import json
-import logging
-import typing as py
 
 log = logging.getLogger(__name__)
 
@@ -155,8 +155,8 @@ def import_records(self, data: py.List[Record]) -> object:
 
 if __name__ == '__main__':
     def _script_io() -> None:
-        from sys import argv
         from os import environ
+        from sys import argv
         from urllib.request import build_opener
 
         main(argv[:], environ.copy(), logging.basicConfig, build_opener())

requirements.txt

@@ -1 +1,4 @@
 requests
+types-requests
+autopep8
+flake8-isort

requirements_pip_freeze.txt

@@ -0,0 +1,17 @@
+autopep8==1.5.7
+certifi==2021.5.30
+chardet==4.0.0
+flake8==3.9.2
+flake8-isort==4.0.0
+idna==2.10
+importlib-metadata==4.6.0
+isort==5.9.1
+mccabe==0.6.1
+pycodestyle==2.7.0
+pyflakes==2.3.1
+requests==2.25.1
+testfixtures==6.17.1
+toml==0.10.2
+typing-extensions==3.10.0.0
+urllib3==1.26.6
+zipp==3.5.0

sds_flat.py

@@ -2,7 +2,6 @@
 import json
 import typing as py
 
-
 RECORD_ID = 'sbjid'
 Record_T = py.Dict[str, str]
 Records_T = py.Iterable[Record_T]

setup.cfg

@@ -0,0 +1,13 @@
+[flake8]
+# autopep, bandit, wemake-python-styleguide(WPS) will use this as well.
+ignore = I900
+#ds_status_sync.py:28:1: I900 'sds_flat' not listed as a requirement
+exclude = .git,venv
+in-place = true
+#for autopep8 to go insid directory
+recursive = true
+
+[isort]
+#sort import
+SKIP = .git,venv
+#verbose=True