Adding e2e_test for Cloud Run image processing sample (GoogleCloudPlatform#4852)

Author: dinagraves

run/image-processing/e2e_test.py

@@ -0,0 +1,161 @@
+# Copyright 2020 Google, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This tests the Pub/Sub image processing sample
+
+import os
+import subprocess
+import time
+import uuid
+
+from google.cloud import storage
+from google.cloud.storage import Blob, notification
+
+import pytest
+
+
+SUFFIX = uuid.uuid4().hex[0:6]
+PROJECT = os.environ['GOOGLE_CLOUD_PROJECT']
+CLOUD_RUN_SERVICE = f"image-proc-{SUFFIX}"
+INPUT_BUCKET = f"image-proc-input-{SUFFIX}"
+OUTPUT_BUCKET = f"image-proc-output-{SUFFIX}"
+
+
+@pytest.fixture()
+def cloud_run_service():
+    # Build and Deploy Cloud Run Services
+    subprocess.run(
+        ["gcloud", "builds", "submit", "--project", PROJECT,
+         f"--substitutions=_SERVICE={CLOUD_RUN_SERVICE},_OUTPUT_BUCKET={OUTPUT_BUCKET}",
+         "--config=e2e_test_setup.yaml", "--quiet"],
+        check=True
+    )
+
+    yield
+
+    # Delete Cloud Run service and image container
+    subprocess.run(
+        ["gcloud", "run", "services", "delete", CLOUD_RUN_SERVICE,
+         "--project", PROJECT, "--platform", "managed", "--region",
+         "us-central1", "--quiet"],
+        check=True
+    )
+
+    subprocess.run(
+        ["gcloud", "container", "images", "delete",
+         f"gcr.io/{PROJECT}/{CLOUD_RUN_SERVICE}", "--quiet"],
+        check=True
+    )
+
+
+@pytest.fixture
+def service_url(cloud_run_service):
+    # Get the URL for the cloud run service
+    service_url = subprocess.run(
+        [
+            "gcloud",
+            "run",
+            "--project",
+            PROJECT,
+            "--platform=managed",
+            "--region=us-central1",
+            "services",
+            "describe",
+            CLOUD_RUN_SERVICE,
+            "--format=value(status.url)",
+        ],
+        stdout=subprocess.PIPE,
+        check=True
+    ).stdout.strip()
+
+    yield service_url.decode()
+
+
+@pytest.fixture()
+def pubsub_topic(service_url):
+    # Create pub/sub topic
+    topic = f"image_proc_{SUFFIX}"
+    subprocess.run(
+        ["gcloud", "pubsub", "topics", "create", topic,
+         "--project", PROJECT, "--quiet"], check=True
+    )
+
+    # Create pubsub push subscription to Cloud Run Service
+    # Attach service account with Cloud Run Invoker role
+    # See tutorial for details on setting up service-account:
+    # https://cloud.google.com/run/docs/tutorials/pubsub
+    subprocess.run(
+        ["gcloud", "pubsub", "subscriptions", "create", f"{topic}_sub",
+         "--topic", topic, "--push-endpoint", service_url, "--project",
+         PROJECT, "--push-auth-service-account",
+         f"cloud-run-invoker@{PROJECT}.iam.gserviceaccount.com", "--quiet"], check=True
+    )
+
+    yield topic
+
+    # Delete topic
+    subprocess.run(
+        ["gcloud", "pubsub", "topics", "delete", topic,
+         "--project", PROJECT, "--quiet"], check=True
+    )
+
+    # Delete subscription
+    subprocess.run(
+        ["gcloud", "pubsub", "subscriptions", "delete", f"{topic}_sub",
+         "--project", PROJECT, "--quiet"], check=True
+    )
+
+
+@pytest.fixture()
+def storage_buckets(pubsub_topic):
+    # Create GCS Buckets
+    storage_client = storage.Client()
+    storage_client.create_bucket(INPUT_BUCKET)
+    storage_client.create_bucket(OUTPUT_BUCKET)
+
+    # Get input and output buckets
+    input_bucket = storage_client.get_bucket(INPUT_BUCKET)
+    output_bucket = storage_client.get_bucket(OUTPUT_BUCKET)
+
+    # Create pub/sub notification on input_bucket
+    notification.BucketNotification(input_bucket, topic_name=pubsub_topic,
+                                    topic_project=PROJECT, payload_format="JSON_API_V1").create()
+
+    yield input_bucket, output_bucket
+
+    # Delete GCS buckets
+    input_bucket.delete(force=True)
+    output_bucket.delete(force=True)
+
+
+def test_end_to_end(storage_buckets):
+    # Upload image to the input bucket
+    input_bucket = storage_buckets[0]
+    output_bucket = storage_buckets[1]
+
+    blob = Blob("zombie.jpg", input_bucket)
+    blob.upload_from_filename("test-images/zombie.jpg", content_type="image/jpeg")
+
+    # Wait for image processing to complete
+    time.sleep(30)
+
+    for x in range(10):
+        # Check for blurred image in output bucket
+        output_blobs = list(output_bucket.list_blobs())
+        if len(output_blobs) > 0:
+            break
+
+        time.sleep(5)
+
+    assert len(output_blobs) > 0

run/image-processing/e2e_test_setup.yaml

@@ -0,0 +1,40 @@
+# Copyright 2020 Google, LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+steps:
+- # Build the renderer image
+  name: gcr.io/cloud-builders/docker:latest
+  args: ['build', '--tag=gcr.io/$PROJECT_ID/${_SERVICE}', '.']
+
+- # Push the container image to Container Registry
+  name: gcr.io/cloud-builders/docker
+  args: ['push', 'gcr.io/$PROJECT_ID/${_SERVICE}']
+
+- # Deploy to Cloud Run
+  name: gcr.io/cloud-builders/gcloud
+  args: 
+  - run
+  - deploy
+  - ${_SERVICE}
+  - --image
+  - gcr.io/$PROJECT_ID/${_SERVICE}
+  - --region
+  - us-central1
+  - --platform
+  - managed
+  - --no-allow-unauthenticated
+  - --set-env-vars=BLURRED_BUCKET_NAME=${_OUTPUT_BUCKET}
+
+images: 
+  - 'gcr.io/$PROJECT_ID/${_SERVICE}'

run/image-processing/noxfile_config.py

@@ -0,0 +1,39 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Default TEST_CONFIG_OVERRIDE for python repos.
+
+# You can copy this file into your directory, then it will be inported from
+# the noxfile.py.
+
+# The source of truth:
+# https://github.com/GoogleCloudPlatform/python-docs-samples/blob/master/noxfile_config.py
+
+TEST_CONFIG_OVERRIDE = {
+    # You can opt out from the test for specific Python versions.
+
+    # We only run the cloud run tests in py38 session.
+    'ignored_versions': ["2.7", "3.6", "3.7"],
+
+    # An envvar key for determining the project id to use. Change it
+    # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a
+    # build specific Cloud project. You can also use your own string
+    # to use your own Cloud project.
+    'gcloud_project_env': 'GOOGLE_CLOUD_PROJECT',
+    # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT',
+
+    # A dictionary you want to inject into your test. Don't put any
+    # secrets here. These values will override predefined values.
+    'envs': {},
+}