diff --git a/docs/apache-airflow-providers-google/operators/cloud/video_intelligence.rst b/docs/apache-airflow-providers-google/operators/cloud/video_intelligence.rst index 2b5d441234ee..8b3f540a9fcb 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/video_intelligence.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/video_intelligence.rst @@ -38,12 +38,12 @@ Using the operator Input uri is an uri to a file in Google Cloud Storage -.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_video_intelligence.py +.. exampleinclude:: /../../tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py :language: python :start-after: [START howto_operator_video_intelligence_other_args] :end-before: [END howto_operator_video_intelligence_other_args] -.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_video_intelligence.py +.. exampleinclude:: /../../tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py :language: python :dedent: 4 :start-after: [START howto_operator_video_intelligence_detect_labels] @@ -51,7 +51,7 @@ Input uri is an uri to a file in Google Cloud Storage You can use the annotation output via Xcom: -.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_video_intelligence.py +.. exampleinclude:: /../../tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py :language: python :dedent: 4 :start-after: [START howto_operator_video_intelligence_detect_labels_result] @@ -87,7 +87,7 @@ Arguments Input uri is an uri to a file in Google Cloud Storage -.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_video_intelligence.py +.. exampleinclude:: /../../tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py :language: python :start-after: [START howto_operator_video_intelligence_other_args] :end-before: [END howto_operator_video_intelligence_other_args] @@ -95,7 +95,7 @@ Input uri is an uri to a file in Google Cloud Storage Using the operator """""""""""""""""" -.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_video_intelligence.py +.. exampleinclude:: /../../tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py :language: python :dedent: 4 :start-after: [START howto_operator_video_intelligence_detect_explicit_content] @@ -103,7 +103,7 @@ Using the operator You can use the annotation output via Xcom: -.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_video_intelligence.py +.. exampleinclude:: /../../tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py :language: python :dedent: 4 :start-after: [START howto_operator_video_intelligence_detect_explicit_content_result] @@ -139,7 +139,7 @@ Arguments Input uri is an uri to a file in Google Cloud Storage -.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_video_intelligence.py +.. exampleinclude:: /../../tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py :language: python :start-after: [START howto_operator_video_intelligence_other_args] :end-before: [END howto_operator_video_intelligence_other_args] @@ -147,7 +147,7 @@ Input uri is an uri to a file in Google Cloud Storage Using the operator """""""""""""""""" -.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_video_intelligence.py +.. exampleinclude:: /../../tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py :language: python :dedent: 4 :start-after: [START howto_operator_video_intelligence_detect_video_shots] @@ -155,7 +155,7 @@ Using the operator You can use the annotation output via Xcom: -.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_video_intelligence.py +.. exampleinclude:: /../../tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py :language: python :dedent: 4 :start-after: [START howto_operator_video_intelligence_detect_video_shots_result] diff --git a/tests/providers/google/cloud/operators/test_video_intelligence_system.py b/tests/providers/google/cloud/operators/test_video_intelligence_system.py deleted file mode 100644 index 8c92bf1e125b..000000000000 --- a/tests/providers/google/cloud/operators/test_video_intelligence_system.py +++ /dev/null @@ -1,49 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -import os - -import pytest - -from airflow.providers.google.cloud.example_dags.example_video_intelligence import GCP_BUCKET_NAME -from tests.providers.google.cloud.utils.gcp_authenticator import GCP_AI_KEY, GCP_GCS_KEY -from tests.test_utils.gcp_system_helpers import CLOUD_DAG_FOLDER, GoogleSystemTest, provide_gcp_context - -GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "example-project") -GCP_VIDEO_SOURCE_URL = "https://www.sample-videos.com/video123/mp4/720/big_buck_bunny_720p_1mb.mp4" - - -@pytest.mark.backend("mysql", "postgres") -@pytest.mark.credential_file(GCP_AI_KEY) -class CloudVideoIntelligenceExampleDagsTest(GoogleSystemTest): - @provide_gcp_context(GCP_AI_KEY) - def setUp(self): - super().setUp() - self.create_gcs_bucket(GCP_BUCKET_NAME, location="europe-north1") - self.execute_with_ctx( - cmd=["bash", "-c", f"curl {GCP_VIDEO_SOURCE_URL} | gsutil cp - gs://{GCP_BUCKET_NAME}/video.mp4"], - key=GCP_GCS_KEY, - ) - - @provide_gcp_context(GCP_AI_KEY) - def tearDown(self): - self.delete_gcs_bucket(GCP_BUCKET_NAME) - super().tearDown() - - @provide_gcp_context(GCP_AI_KEY) - def test_example_dag(self): - self.run_dag('example_gcp_video_intelligence', CLOUD_DAG_FOLDER) diff --git a/tests/system/providers/google/cloud/video_intelligence/__init__.py b/tests/system/providers/google/cloud/video_intelligence/__init__.py new file mode 100644 index 000000000000..13a83393a912 --- /dev/null +++ b/tests/system/providers/google/cloud/video_intelligence/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/airflow/providers/google/cloud/example_dags/example_video_intelligence.py b/tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py similarity index 67% rename from airflow/providers/google/cloud/example_dags/example_video_intelligence.py rename to tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py index 9e8183e78545..9a214e3d28f2 100644 --- a/airflow/providers/google/cloud/example_dags/example_video_intelligence.py +++ b/tests/system/providers/google/cloud/video_intelligence/example_video_intelligence.py @@ -22,7 +22,7 @@ This DAG relies on the following OS environment variables: -* GCP_BUCKET_NAME - Google Cloud Storage bucket where the file exists. +* BUCKET_NAME - Google Cloud Storage bucket where the file exists. """ import os from datetime import datetime @@ -30,30 +30,53 @@ from google.api_core.retry import Retry from airflow import models +from airflow.models.baseoperator import chain from airflow.operators.bash import BashOperator +from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator from airflow.providers.google.cloud.operators.video_intelligence import ( CloudVideoIntelligenceDetectVideoExplicitContentOperator, CloudVideoIntelligenceDetectVideoLabelsOperator, CloudVideoIntelligenceDetectVideoShotsOperator, ) +from airflow.providers.google.cloud.transfers.gcs_to_gcs import GCSToGCSOperator +from airflow.utils.trigger_rule import TriggerRule + +ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID") + +DAG_ID = "example_gcp_video_intelligence" + +# Public bucket holding the sample data +BUCKET_NAME_SRC = "cloud-samples-data" +# Path to the data inside the public bucket +PATH_SRC = "video/cat.mp4" # [START howto_operator_video_intelligence_os_args] -GCP_BUCKET_NAME = os.environ.get("GCP_VIDEO_INTELLIGENCE_BUCKET_NAME", "INVALID BUCKET NAME") +BUCKET_NAME_DST = f"bucket-src-{DAG_ID}-{ENV_ID}" # [END howto_operator_video_intelligence_os_args] +FILE_NAME = "video.mp4" # [START howto_operator_video_intelligence_other_args] -INPUT_URI = f"gs://{GCP_BUCKET_NAME}/video.mp4" +INPUT_URI = f"gs://{BUCKET_NAME_DST}/{FILE_NAME}" # [END howto_operator_video_intelligence_other_args] - with models.DAG( - "example_gcp_video_intelligence", + DAG_ID, start_date=datetime(2021, 1, 1), catchup=False, tags=['example'], ) as dag: + create_bucket = GCSCreateBucketOperator(task_id="create_bucket", bucket_name=BUCKET_NAME_DST) + + copy_single_file = GCSToGCSOperator( + task_id="copy_single_gcs_file", + source_bucket=BUCKET_NAME_SRC, + source_object=PATH_SRC, + destination_bucket=BUCKET_NAME_DST, + destination_object=FILE_NAME, + ) + # [START howto_operator_video_intelligence_detect_labels] detect_video_label = CloudVideoIntelligenceDetectVideoLabelsOperator( input_uri=INPUT_URI, @@ -110,6 +133,33 @@ ) # [END howto_operator_video_intelligence_detect_video_shots_result] - detect_video_label >> detect_video_label_result - detect_video_explicit_content >> detect_video_explicit_content_result - detect_video_shots >> detect_video_shots_result + delete_bucket = GCSDeleteBucketOperator( + task_id="delete_bucket", bucket_name=BUCKET_NAME_DST, trigger_rule=TriggerRule.ALL_DONE + ) + + chain( + # TEST SETUP + create_bucket, + copy_single_file, + # TEST BODY + detect_video_label, + detect_video_label_result, + detect_video_explicit_content, + detect_video_explicit_content_result, + detect_video_shots, + detect_video_shots_result, + # TEST TEARDOWN + delete_bucket, + ) + + from tests.system.utils.watcher import watcher + + # This test needs watcher in order to properly mark success/failure + # when "tearDown" task with trigger rule is part of the DAG + list(dag.tasks) >> watcher() + + +from tests.system.utils import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +test_run = get_test_run(dag)