Merge pull request #331 from rstudio/vetiver-tests

Vetiver tests

Author: bcwu

.github/workflows/main.yml

@@ -146,3 +146,29 @@ jobs:
             repo: 'rsconnect-jupyter',
             event_type: 'rsconnect_python_latest'
           })
+  test-rsconnect:
+    name: "Test vetiver"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -r requirements.txt
+          python -m pip install -r vetiver-testing/vetiver-requirements.txt
+      - name: run RStudio Connect
+        run: |
+          docker-compose up --build -d
+          pip freeze > requirements.txt
+          make dev
+        env:
+          RSC_LICENSE: ${{ secrets.RSC_LICENSE }}
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+
+      # NOTE: edited to run checks for python package
+      - name: Run tests
+        run: |
+          pytest -m 'vetiver'

Makefile

@@ -20,6 +20,8 @@ endif
 
 TEST_ENV =
 
+RSC_API_KEYS=vetiver-testing/rsconnect_api_keys.json
+
 ifneq ($(CONNECT_SERVER),)
   TEST_ENV += CONNECT_SERVER=$(CONNECT_SERVER)
 endif
@@ -165,3 +167,21 @@ promote-docs-in-s3:
 		--cache-control max-age=300 \
 		docs/site/ \
 		s3://docs.rstudio.com/rsconnect-python/
+
+
+dev: vetiver-testing/rsconnect_api_keys.json
+
+dev-start:
+	docker-compose up -d
+	docker-compose exec -T rsconnect bash < vetiver-testing/setup-rsconnect/add-users.sh
+	# curl fails with error 52 without a short sleep....
+	sleep 5
+	curl -s --retry 10 --retry-connrefused http://localhost:3939
+
+dev-stop:
+	docker-compose down
+	rm -f $(RSC_API_KEYS)
+
+$(RSC_API_KEYS): dev-start
+	python vetiver-testing/setup-rsconnect/dump_api_keys.py $@
+	

conftest.py

@@ -1,7 +1,22 @@
 import sys
+import pytest
 
 from os.path import abspath, dirname
 
 
 HERE = dirname(abspath(__file__))
 sys.path.insert(0, HERE)
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--vetiver", action="store_true", default=False, help="run vetiver tests"
+    )
+
+def pytest_configure(config):
+    config.addinivalue_line("markers", "vetiver: test for vetiver interaction")
+
+def pytest_collection_modifyitems(config, items):
+    if config.getoption("--vetiver"):
+        return
+    skip_vetiver = pytest.mark.skip(reason="need --vetiver option to run")

docker-compose.yml

@@ -0,0 +1,17 @@
+version: '3.2'
+
+services:
+
+  rsconnect:
+    image: rstudio/rstudio-connect:latest
+    restart: always
+    ports:
+      - 3939:3939
+    volumes:
+      - $PWD/vetiver-testing/setup-rsconnect/users.txt:/etc/users.txt
+      - $PWD/vetiver-testing/setup-rsconnect/rstudio-connect.gcfg:/etc/rstudio-connect/rstudio-connect.gcfg
+    # by default, mysql rounds to 4 decimals, but tests require more precision
+    privileged: true
+    environment:
+      RSTUDIO_CONNECT_HASTE: "enabled"
+      RSC_LICENSE: ${RSC_LICENSE}

pyproject.toml

@@ -10,3 +10,8 @@ omit = ["tests/*"]
 
 [tool.setuptools_scm]
 write_to = "rsconnect/version.py"
+
+[tool.pytest.ini_options]
+markers = [
+    "vetiver: tests for vetiver",
+]

tests/test_vetiver_pins.py

@@ -0,0 +1,95 @@
+import pytest
+
+vetiver = pytest.importorskip("vetiver", reason="vetiver library not installed")
+
+import json # noqa
+import pins # noqa
+import pandas as pd # noqa
+import numpy as np # noqa
+
+from pins.boards import BoardRsConnect  # noqa
+from pins.rsconnect.api import RsConnectApi  # noqa
+from pins.rsconnect.fs import RsConnectFs  # noqa
+from rsconnect.api import RSConnectServer, RSConnectClient # noqa
+
+RSC_SERVER_URL = "http://localhost:3939"
+RSC_KEYS_FNAME = "vetiver-testing/rsconnect_api_keys.json"
+
+pytestmark = pytest.mark.vetiver  # noqa
+
+
+def get_key(name):
+    with open(RSC_KEYS_FNAME) as f:
+        api_key = json.load(f)[name]
+        return api_key
+
+
+def rsc_from_key(name):
+    with open(RSC_KEYS_FNAME) as f:
+        api_key = json.load(f)[name]
+        return RsConnectApi(RSC_SERVER_URL, api_key)
+
+
+def rsc_fs_from_key(name):
+
+    rsc = rsc_from_key(name)
+
+    return RsConnectFs(rsc)
+
+
+def rsc_delete_user_content(rsc):
+    guid = rsc.get_user()["guid"]
+    content = rsc.get_content(owner_guid=guid)
+    for entry in content:
+        rsc.delete_content_item(entry["guid"])
+
+
+@pytest.fixture(scope="function")
+def rsc_short():
+    # tears down content after each test
+    fs_susan = rsc_fs_from_key("susan")
+
+    # delete any content that might already exist
+    rsc_delete_user_content(fs_susan.api)
+
+    yield BoardRsConnect("", fs_susan, allow_pickle_read=True)  # fs_susan.ls to list content
+
+    rsc_delete_user_content(fs_susan.api)
+
+
+def test_deploy(rsc_short):
+    np.random.seed(500)
+
+    # Load data, model
+    X_df, y = vetiver.mock.get_mock_data()
+    model = vetiver.mock.get_mock_model().fit(X_df, y)
+
+    v = vetiver.VetiverModel(model=model, ptype_data=X_df, model_name="susan/model")
+
+    board = pins.board_rsconnect(server_url=RSC_SERVER_URL, api_key=get_key("susan"), allow_pickle_read=True)
+
+    vetiver.vetiver_pin_write(board=board, model=v)
+    connect_server = RSConnectServer(url=RSC_SERVER_URL, api_key=get_key("susan"))
+
+    vetiver.deploy_rsconnect(
+        connect_server=connect_server,
+        board=board,
+        pin_name="susan/model",
+        title="testapivetiver",
+        extra_files=["requirements.txt"],
+    )
+
+    # get url of where content lives
+    client = RSConnectClient(connect_server)
+    dicts = client.content_search()
+    rsc_api = list(filter(lambda x: x["title"] == "testapivetiver", dicts))
+    content_url = rsc_api[0].get("content_url")
+
+    h = {"Authorization": 'Key {}'.format(get_key("susan"))}
+
+    endpoint = vetiver.vetiver_endpoint(content_url + "/predict")
+    response = vetiver.predict(endpoint, X_df, headers=h)
+
+    assert isinstance(response, pd.DataFrame), response
+    assert response.iloc[0, 0] == 44.47
+    assert len(response) == 100

vetiver-testing/setup-rsconnect/add-users.sh

@@ -0,0 +1 @@
+awk ' { system("useradd -m -s /bin/bash "$1); system("echo \""$1":"$2"\" | chpasswd"); system("id "$1) } ' /etc/users.txt

vetiver-testing/setup-rsconnect/dump_api_keys.py

@@ -0,0 +1,21 @@
+import json
+import sys
+
+from pins.rsconnect.api import _HackyConnect
+
+OUT_FILE = sys.argv[1]
+
+
+def get_api_key(user, password, email):
+    rsc = _HackyConnect("http://localhost:3939")
+
+    return rsc.create_first_admin(user, password, email).api_key
+
+
+api_keys = {
+    "admin": get_api_key("admin", "admin0", "admin@example.com"),
+    "susan": get_api_key("susan", "susan", "susan@example.com"),
+    "derek": get_api_key("derek", "derek", "derek@example.com"),
+}
+
+json.dump(api_keys, open(OUT_FILE, "w"))

vetiver-testing/setup-rsconnect/rstudio-connect.gcfg

@@ -0,0 +1,23 @@
+[Server]
+DataDir = /data
+Address = http://localhost:3939
+
+[HTTP]
+Listen = :3939
+
+[Authentication]
+Provider = pam
+
+[Authorization]
+DefaultUserRole = publisher
+
+[Python]
+Enabled = true
+Executable = /opt/python/3.8.10/bin/python
+Executable = /opt/python/3.9.5/bin/python
+
+[RPackageRepository "CRAN"]
+URL = https://packagemanager.rstudio.com/cran/__linux__/bionic/latest
+
+[RPackageRepository "RSPM"]
+URL = https://packagemanager.rstudio.com/cran/__linux__/bionic/latest

vetiver-testing/setup-rsconnect/users.txt

@@ -0,0 +1,4 @@
+admin admin0
+test  test
+susan susan
+derek derek

vetiver-testing/vetiver-requirements.txt

@@ -0,0 +1,5 @@
+pins
+pandas
+numpy
+vetiver
+pytest