Skip to content

Commit

Permalink
fix test (#376)
Browse files Browse the repository at this point in the history
  • Loading branch information
zubenkoivan authored Apr 12, 2021
1 parent dbe2cb6 commit 867b84f
Show file tree
Hide file tree
Showing 7 changed files with 231 additions and 114 deletions.
5 changes: 4 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ CLOUD_IMAGE = ${CLOUD_IMAGE_${CLOUD_PROVIDER}}
PLATFORMAPI_IMAGE = $(shell cat PLATFORMAPI_IMAGE)
PLATFORMAUTHAPI_IMAGE = $(shell cat PLATFORMAUTHAPI_IMAGE)
PLATFORMCONFIG_IMAGE = $(shell cat PLATFORMCONFIG_IMAGE)
PLATFORMNOTIFICATIONS_IMAGE = $(shell cat PLATFORMNOTIFICATIONS_IMAGE)

ARTIFACTORY_DOCKER_REPO ?= neuro-docker-local-public.jfrog.io
ARTIFACTORY_HELM_REPO ?= https://neuro.jfrog.io/artifactory/helm-local-public
Expand Down Expand Up @@ -84,9 +85,11 @@ docker_pull_test_images:
docker pull $(PLATFORMAPI_IMAGE); \
docker pull $(PLATFORMAUTHAPI_IMAGE); \
docker pull $(PLATFORMCONFIG_IMAGE); \
docker pull $(PLATFORMNOTIFICATIONS_IMAGE); \
docker tag $(PLATFORMAPI_IMAGE) platformapi:latest; \
docker tag $(PLATFORMAUTHAPI_IMAGE) platformauthapi:latest; \
docker tag $(PLATFORMCONFIG_IMAGE) platformconfig:latest
docker tag $(PLATFORMCONFIG_IMAGE) platformconfig:latest; \
docker tag $(PLATFORMNOTIFICATIONS_IMAGE) platformnotifications:latest

gcr_login:
@echo $(GKE_ACCT_AUTH) | base64 --decode | docker login -u _json_key --password-stdin https://gcr.io
Expand Down
1 change: 1 addition & 0 deletions PLATFORMNOTIFICATIONS_IMAGE
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
crc570d91c95c6aac0ea80afb1019a0c6f.azurecr.io/platformnotificationsapi:latest
99 changes: 6 additions & 93 deletions tests/integration/conftest_config.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,7 @@
import asyncio
from typing import Any, AsyncIterator, Callable, Dict
from typing import AsyncIterator, Callable

import aiohttp
import pytest
from aiohttp.web_exceptions import HTTPCreated, HTTPNoContent
from platform_config_client import ConfigClient
from yarl import URL

Expand All @@ -13,68 +11,16 @@


@pytest.fixture
def cluster_name(_cluster: str) -> str:
return _cluster
async def cluster_name(platform_api_config: PlatformApiConfig) -> str:
await asyncio.wait_for(_wait_for_platform_api_config(platform_api_config), 30)
return "default"


@pytest.fixture(scope="session")
def cluster_token(token_factory: Callable[[str], str]) -> str:
return token_factory("cluster")


@pytest.fixture
def _cluster_payload() -> Dict[str, Any]:
return {
"name": "default",
"storage": {
"host": {"mount_path": "/tmp"},
"url": "http://platformapi/api/v1/storage",
},
"blob_storage": {"url": "http://platformapi/api/v1/blob"},
"registry": {
"url": "http://localhost:5000",
"email": "registry@neuromation.io",
},
"orchestrator": {
"kubernetes": {
"url": "http://localhost:8001",
"auth_type": "none",
"namespace": "default",
"node_label_gpu": "cloud.google.com/gke-accelerator",
"node_label_preemptible": "cloud.google.com/gke-preemptible",
"node_label_node_pool": "platform.neuromation.io/nodepool",
"node_label_job": "platform.neuromation.io/job",
},
"is_http_ingress_secure": True,
"job_hostname_template": "{job_id}.jobs.neu.ro",
"resource_pool_types": [
{
"name": "minikube",
"min_size": 1,
"max_size": 1,
"cpu": 1.0,
"available_cpu": 1.0,
"memory_mb": 1024,
"available_memory_mb": 1024,
}
],
"resource_presets": [
{
"name": "cpu-small",
"credits_per_hour": "0.0",
"cpu": 0.1,
"memory_mb": 100,
}
],
},
"monitoring": {"url": "http://platformapi/api/v1/jobs"},
"secrets": {"url": "http://platformapi/api/v1/secrets"},
"metrics": {"url": "http://platformapi/api/v1/metrics"},
"disks": {"url": "http://platformapi/api/v1/disk"},
"ingress": {"acme_environment": "staging"},
}


@pytest.fixture
def platform_config_url(in_minikube: bool) -> URL:
if in_minikube:
Expand All @@ -90,47 +36,14 @@ def platform_config(


@pytest.fixture
@pytest.mark.usefixtures("cluster_name")
async def platform_config_client(
platform_config_url: URL, cluster_name: str, cluster_token: str
platform_config_url: URL, cluster_token: str
) -> AsyncIterator[ConfigClient]:
async with ConfigClient(url=platform_config_url, token=cluster_token) as client:
yield client


@pytest.fixture
async def _cluster(
client: aiohttp.ClientSession,
platform_api_config: PlatformApiConfig,
platform_config_url: URL,
cluster_token: str,
_cluster_payload: Dict[str, Any],
) -> AsyncIterator[str]:
cluster_name = _cluster_payload["name"]
try:
await client.delete(
platform_config_url / "api/v1/clusters" / cluster_name,
headers={"Authorization": f"Bearer {cluster_token}"},
)
except Exception:
pass
try:
response = await client.post(
platform_config_url / "api/v1/clusters",
headers={"Authorization": f"Bearer {cluster_token}"},
json=_cluster_payload,
)
assert response.status == HTTPCreated.status_code, await response.text()
await asyncio.wait_for(_wait_for_platform_api_config(platform_api_config), 30)
yield cluster_name
finally:
response = await client.delete(
platform_config_url / "api/v1/clusters" / cluster_name,
headers={"Authorization": f"Bearer {cluster_token}"},
)
response_text = await response.text()
assert response.status == HTTPNoContent.status_code, response_text


async def _wait_for_platform_api_config(
platform_api_config: PlatformApiConfig, sleep_s: float = 0.1
) -> None:
Expand Down
1 change: 1 addition & 0 deletions tests/k8s/cluster.sh
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ function k8s::apply_all_configurations {
kubectl apply -f tests/k8s/logging.yml
kubectl apply -f tests/k8s/platformconfig.yml
kubectl apply -f tests/k8s/platformapi.yml
kubectl apply -f tests/k8s/platformnotifications.yml
kubectl apply -f tests/k8s/dockerengineapi.yml
}

Expand Down
55 changes: 37 additions & 18 deletions tests/k8s/platformapi.yml
Original file line number Diff line number Diff line change
Expand Up @@ -83,28 +83,39 @@ spec:
service: platformapi
spec:
initContainers:
- name: wait-for-cluster
image: curlimages/curl
command: [sh]
args:
- -c
- |
code=-1
while [ $code != 0 ]
do
curl -f -H "Authorization: Bearer $NP_TOKEN" "$NP_PLATFORM_CONFIG_URL/api/v1/clusters/default"
code=$?
done
env:
- name: NP_PLATFORM_CONFIG_URL
value: http://platformconfig:8080
- name: NP_TOKEN
value: eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpZGVudGl0eSI6ImNvbXB1dGUifQ.IJDlKCfbiuNGZH9Sh6p-CdUL5KxEK5JStqzfDk4-RAA
- name: platformapi-migrations
image: platformapi:latest
imagePullPolicy: Never
command: ["alembic", "upgrade", "head"]
env:
- name: NP_DB_POSTGRES_DSN
value: postgresql://postgres:postgres@platformapi-postgres:5432
env:
- name: NP_DB_POSTGRES_DSN
value: postgresql://postgres:postgres@platformapi-postgres:5432
containers:
- name: platformapi
image: platformapi:latest
imagePullPolicy: Never
env:
- name: NP_STORAGE_HOST_MOUNT_PATH
value: /tmp
- name: NP_K8S_JOBS_INGRESS_NAME
value: not-important-NP_K8S_JOBS_INGRESS_NAME
- name: NP_K8S_JOBS_INGRESS_DOMAIN_NAME_TEMPLATE
value: not-important-NP_K8S_JOBS_INGRESS_DOMAIN_NAME_TEMPLATE
- name: NP_K8S_NAMED_JOBS_INGRESS_DOMAIN_NAME_TEMPLATE
value: not-important-NP_K8S_NAMED_JOBS_INGRESS_DOMAIN_NAME_TEMPLATE
- name: NP_K8S_SSH_AUTH_INGRESS_DOMAIN_NAME
value: not-important-NP_K8S_SSH_AUTH_INGRESS_DOMAIN_NAME
- name: NP_OAUTH_AUTH_URL
value: not-important-NP_OAUTH_AUTH_URL
- name: NP_OAUTH_TOKEN_URL
Expand All @@ -126,9 +137,9 @@ spec:
- name: NP_ENFORCER_INTERVAL_SEC
value: "100500"
- name: NP_NOTIFICATIONS_URL
value: not-important-NP_NOTIFICATIONS_URL
value: http://platformnotifications:8080
- name: NP_NOTIFICATIONS_TOKEN
value: not-important-NP_NOTIFICATIONS_TOKEN
value: eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpZGVudGl0eSI6ImNvbXB1dGUifQ.IJDlKCfbiuNGZH9Sh6p-CdUL5KxEK5JStqzfDk4-RAA
- name: NP_API_ZIPKIN_URL
value: not-important-NP_API_ZIPKIN_URL
- name: NP_API_ZIPKIN_SAMPLE_RATE
Expand All @@ -137,10 +148,6 @@ spec:
value: http://platformauthapi:8080
- name: NP_AUTH_TOKEN
value: eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpZGVudGl0eSI6ImNvbXB1dGUifQ.IJDlKCfbiuNGZH9Sh6p-CdUL5KxEK5JStqzfDk4-RAA
- name: NP_ES_HOSTS
value: http://elasticsearch-logging.kube-system:9200
- name: NP_JOBS_INGRESS_OAUTH_AUTHORIZE_URL
value: not-important-NP_JOBS_INGRESS_OAUTH_AUTHORIZE_URL
- name: NP_ADMIN_URL
value: not-important-NP_ADMIN_URL
- name: NP_API_URL
Expand Down Expand Up @@ -178,14 +185,26 @@ spec:
value: eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpZGVudGl0eSI6ImNvbXB1dGUifQ.IJDlKCfbiuNGZH9Sh6p-CdUL5KxEK5JStqzfDk4-RAA
- name: NP_PLATFORM_API_URL
value: http://platformapi:8080/api/v1
- name: NP_JOBS_INGRESS_OAUTH_AUTHORIZE_URL
value: not-important-NP_JOBS_INGRESS_OAUTH_AUTHORIZE_URL
- name: NP_PLATFORM_CONFIG_URI
value: http://platformconfig:8080/api/v1
- name: NP_AUTH_PUBLIC_URL
value: http://platformauthapi:8080
- name: NP_AUTH_TOKEN
value: eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpZGVudGl0eSI6ImNvbXB1dGUifQ.IJDlKCfbiuNGZH9Sh6p-CdUL5KxEK5JStqzfDk4-RAA
- name: NP_CLUSTER_NAME
value: default
- name: NP_KUBE_URL
value: http://localhost:8001
- name: NP_KUBE_INGRESS_CLASS
value: not-important
- name: NP_KUBE_INGRESS_OAUTH_AUTHORIZE_URL
value: not-important-NP_JOBS_INGRESS_OAUTH_AUTHORIZE_URL
- name: NP_REGISTRY_URL
value: http://not-important
- name: NP_STORAGE_TYPE
value: host
- name: NP_HOST_MOUNT_PATH
value: /tmp
- name: kubeproxy
image: lachlanevenson/k8s-kubectl:v1.10.3
args: [proxy]
Expand Down
82 changes: 80 additions & 2 deletions tests/k8s/platformconfig.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,6 @@ spec:
value: "8080"
- name: NP_PLATFORM_AUTH_URL
value: http://platformauthapi:8080
- name: NP_PLATFORM_ADMIN_URL
value: http://platformadmin:8080
- name: NP_PLATFORM_CONFIG_TOKEN
value: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZGVudGl0eSI6ImNsdXN0ZXIifQ.kZG6EyaHMa2gNzrgKQlwVnZ0OZ5mruFZ0xVAbntm-dU
- name: NP_PUBLIC_NEURO_REGISTRY_URL
Expand All @@ -60,6 +58,8 @@ spec:
value: not-important
- name: NP_AWS_SECRET_ACCESS_KEY
value: not-important
- name: NP_AWS_ROUTE53_MAIN_ZONE_ID
value: not-important
- name: NP_CLUSTER_DNS_SUFFIX
value: not-important
- name: NP_K8S_URL
Expand Down Expand Up @@ -137,3 +137,81 @@ spec:
- port: 5432
selector:
service: platformconfig-postgres
---
apiVersion: batch/v1
kind: Job
metadata:
name: create-cluster
spec:
template:
spec:
restartPolicy: Never
initContainers:
- name: wait-for-platform-config
image: curlimages/curl
command: [sh]
args:
- -c
- |
code=-1
while [ $code != 0 ]
do
curl -f "$NP_PLATFORM_CONFIG_URL/healthchecks/liveness"
code=$?
done
env:
- name: NP_PLATFORM_CONFIG_URL
value: http://platformconfig:8080
containers:
- name: create-cluster
image: curlimages/curl
command: [sh]
args:
- -c
- |
curl -vv -X POST \
-H "Authorization: Bearer $NP_TOKEN" \
-d @- \
"$NP_PLATFORM_CONFIG_URL/api/v1/clusters" <<EOM
{
"name": "default",
"orchestrator": {
"is_http_ingress_secure": true,
"job_hostname_template": "{job_id}.jobs.neu.ro",
"job_internal_hostname_template": "{job_id}.default",
"resource_pool_types": [
{
"name": "minikube",
"min_size": 1,
"max_size": 1,
"cpu": 1.0,
"available_cpu": 1.0,
"memory_mb": 1024,
"available_memory_mb": 1024
}
],
"resource_presets": [
{
"name": "cpu-small",
"credits_per_hour": "0.0",
"cpu": 0.1,
"memory_mb": 100
}
]
},
"storage": {"url": "http://platformapi/api/v1/storage"},
"blob_storage": {"url": "http://platformapi/api/v1/blob"},
"registry": {"url": "http://localhost:5000"},
"monitoring": {"url": "http://platformapi/api/v1/jobs"},
"secrets": {"url": "http://platformapi/api/v1/secrets"},
"metrics": {"url": "http://platformapi/api/v1/metrics"},
"disks": {"url": "http://platformapi/api/v1/disk"},
"ingress": {"acme_environment": "staging"}
}
EOM
env:
- name: NP_PLATFORM_CONFIG_URL
value: http://platformconfig:8080
- name: NP_TOKEN
value: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZGVudGl0eSI6ImNsdXN0ZXIifQ.kZG6EyaHMa2gNzrgKQlwVnZ0OZ5mruFZ0xVAbntm-dU
Loading

0 comments on commit 867b84f

Please sign in to comment.