diff --git a/.gitignore b/.gitignore index 2683466..e1d6d44 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,6 @@ .direnv .envrc __pycache__ -.mypy_cache \ No newline at end of file +.mypy_cache +venv +out/ diff --git a/Makefile b/Makefile index 3293a33..bc5f4db 100644 --- a/Makefile +++ b/Makefile @@ -4,12 +4,27 @@ FLAKE8=flake8 SRCS=ref_code_gen.py ds_status_sync.py sds_flat.py -check: doctest lint static +check: autopep autoblack lint doctest lint static + +autopep: + # "converting python code to PEP formate" + autopep8 . + +autoblack: + # "converting python code to BLACK formate" + #black . static: - $(MYPY) --strict $(SRCS) + # ds_status_sync.py:311: error: unused "type: ignore" comment + # ds_status_sync.py:343: error: unused "type: ignore" comment + # using --no-warn-no-return to hide above error + $(MYPY) --strict $(SRCS) --no-warn-no-return -lint: $(SRCS) +lint-import: + # this does import sorting for you, where flak8-isort will check it for you + isort . + +lint: $(SRCS) lint-import $(FLAKE8) $(SRCS) doctest: @@ -38,3 +53,25 @@ debug: lint check install-dev-tools: pip install mypy flake8 + +ds_sync: clean venv + . venv/bin/activate && \ + which python && python --version &&\ + python ds_status_sync.py --get-status REDCAP_API_TOKEN DS_KEY >out/ds_status.json \ + python ds_status_sync.py --send-consent REDCAP_API_TOKEN DS_KEY + +clean: + rm -rf out; mkdir -p out + +venv: venv_clear + # "creating python virtual env" + python -m venv venv + . ./venv/bin/activate && \ + pip install --upgrade pip && \ + pip install -r requirements.txt && \ + pip freeze > requirements_pip_freeze.txt && \ + which pip && which python && python --version + +venv_clear: + # "deleting python virtual env" + rm -rf venv || true diff --git a/ds_status_sync.py b/ds_status_sync.py index 61158d6..b0678e0 100644 --- a/ds_status_sync.py +++ b/ds_status_sync.py @@ -22,9 +22,10 @@ from pathlib import Path as Path_T from urllib.error import HTTPError -from requests import Request, Session as Session_T +from requests import Request +from requests import Session as Session_T -from sds_flat import flatten, complete +from sds_flat import complete, flatten log = logging.getLogger(__name__) @@ -36,6 +37,50 @@ WebBuilder = py.Callable[..., Session_T] Record_T = py.Dict[str, str] +selected_survey_key = ["record_id", + "registerdate", + "lastvisitdate", + "lastupdatedate", + "s_1_title", + "s_1_completed", + "s_1_played_time", + "s_2_title", + "s_2_completed", + "s_2_played_time", + "s_3_title", + "s_3_completed", + "s_3_played_time", + "s_4_title", + "s_4_completed", + "s_4_played_time", + "s_5_title", + "s_5_completed", + "s_5_played_time", + "s_6_title", + "s_6_completed", + "s_6_played_time", + "s_7_title", + "s_7_completed", + "s_7_played_time", + "s_8_title", + "s_8_completed", + "s_8_played_time", + "ds_connect_status_complete"] + + +def select_recrods_surveys(records: py.List[Record_T], + selected_survey_key: py.List[str]) \ + -> py.List[Record_T]: + output_records = [] + + for record in records: + output_record = {} + for key in record: + if key in selected_survey_key: + output_record[key] = record[key] + output_records.append(output_record) + return output_records + def main(argv: py.List[str], env: py.Dict[str, str], stdout: py.IO[str], cwd: Path_T, now: py.Callable[[], datetime], @@ -55,6 +100,7 @@ def study(api_passkey: str) -> DSConnectStudy: rc = REDCapAPI(REDCAP_API, make_session(), env[api_passkey]) status = ds.getstatus([DS_DETERMINED]) records = list(complete(STATUS_FORM)(flatten(status))) + records = select_recrods_surveys(records, selected_survey_key) json.dump({'status': status, 'records': records}, stdout, indent=2) rc.import_records(records) elif '--send-consent' in argv: @@ -368,7 +414,7 @@ def _script_io() -> None: from datetime import datetime from os import environ from pathlib import Path - from sys import argv, stdout, stderr + from sys import argv, stderr, stdout from requests import Session diff --git a/ref_code_gen.py b/ref_code_gen.py index 8e34894..c71363c 100644 --- a/ref_code_gen.py +++ b/ref_code_gen.py @@ -21,14 +21,14 @@ {'record_id': 'SC-_TEST_00014', 'redcap_data_access_group': 'sc'}] """ +import json +import logging +import typing as py from binascii import crc32 from pprint import pformat from urllib.error import HTTPError from urllib.parse import urlencode from urllib.request import OpenerDirector as OpenerDirector_T -import json -import logging -import typing as py log = logging.getLogger(__name__) @@ -155,8 +155,8 @@ def import_records(self, data: py.List[Record]) -> object: if __name__ == '__main__': def _script_io() -> None: - from sys import argv from os import environ + from sys import argv from urllib.request import build_opener main(argv[:], environ.copy(), logging.basicConfig, build_opener()) diff --git a/requirements.txt b/requirements.txt index f229360..5135001 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,4 @@ requests +types-requests +autopep8 +flake8-isort diff --git a/requirements_pip_freeze.txt b/requirements_pip_freeze.txt new file mode 100644 index 0000000..cc1dc5e --- /dev/null +++ b/requirements_pip_freeze.txt @@ -0,0 +1,17 @@ +autopep8==1.5.7 +certifi==2021.5.30 +chardet==4.0.0 +flake8==3.9.2 +flake8-isort==4.0.0 +idna==2.10 +importlib-metadata==4.6.0 +isort==5.9.1 +mccabe==0.6.1 +pycodestyle==2.7.0 +pyflakes==2.3.1 +requests==2.25.1 +testfixtures==6.17.1 +toml==0.10.2 +typing-extensions==3.10.0.0 +urllib3==1.26.6 +zipp==3.5.0 diff --git a/sds_flat.py b/sds_flat.py index d8307da..3721a9a 100644 --- a/sds_flat.py +++ b/sds_flat.py @@ -2,7 +2,6 @@ import json import typing as py - RECORD_ID = 'sbjid' Record_T = py.Dict[str, str] Records_T = py.Iterable[Record_T] diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..5e47434 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,13 @@ +[flake8] +# autopep, bandit, wemake-python-styleguide(WPS) will use this as well. +ignore = I900 +#ds_status_sync.py:28:1: I900 'sds_flat' not listed as a requirement +exclude = .git,venv +in-place = true +#for autopep8 to go insid directory +recursive = true + +[isort] +#sort import +SKIP = .git,venv +#verbose=True