Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 19 additions & 17 deletions composer/workflows/kubernetes_pod_operator_c2.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@
schedule_interval=datetime.timedelta(days=1),
start_date=YESTERDAY,
) as dag:
# Only name, namespace, image, and task_id are required to create a
# Only name, image, and task_id are required to create a
# KubernetesPodOperator. In Cloud Composer, the config file found at
# `/home/airflow/composer_kube_config` contains credentials for
# Cloud Composer's Google Kubernetes Engine cluster that is created
Expand All @@ -79,7 +79,8 @@
cmds=["echo"],
# The namespace to run within Kubernetes. In Composer 2 environments
# after December 2022, the default namespace is
# `composer-user-workloads`.
# `composer-user-workloads`. Always use the
# `composer-user-workloads` namespace with Composer 3.
namespace="composer-user-workloads",
# Docker image specified. Defaults to hub.docker.com, but any fully
# qualified URLs will point to a custom repository. Supports private
Expand All @@ -100,27 +101,26 @@
name="ex-kube-templates",
namespace="composer-user-workloads",
image="bash",
# All parameters below are able to be templated with jinja -- cmds,
# arguments, env_vars, and config_file. For more information visit:
# https://airflow.apache.org/docs/apache-airflow/stable/macros-ref.html
# All parameters below can be templated with Jinja. For more information
# and the list of variables available in Airflow, see
# the Airflow templates reference:
# https://airflow.apache.org/docs/apache-airflow/stable/templates-ref.html
# Entrypoint of the container, if not specified the Docker container's
# entrypoint is used. The cmds parameter is templated.
cmds=["echo"],
# DS in jinja is the execution date as YYYY-MM-DD, this docker image
# will echo the execution date. Arguments to the entrypoint. The docker
# DS in Jinja is the execution date as YYYY-MM-DD, this Docker image
# will echo the execution date. Arguments to the entrypoint. The Docker
# image's CMD is used if this is not provided. The arguments parameter
# is templated.
arguments=["{{ ds }}"],
# The var template variable allows you to access variables defined in
# Airflow UI. In this case we are getting the value of my_value and
# setting the environment variable `MY_VALUE`. The pod will fail if
# `my_value` is not set in the Airflow UI.
# `my_value` is not set in the Airflow UI. The env_vars parameter
# is templated.
env_vars={"MY_VALUE": "{{ var.value.my_value }}"},
# Sets the config file to a kubernetes config file specified in
# airflow.cfg. If the configuration file does not exist or does
# not provide validcredentials the pod will fail to launch. If not
# specified, config_file defaults to ~/.kube/config
config_file="{{ conf.get('core', 'kube_config') }}",
# Specifies path to Kubernetes config. The config_file is templated.
config_file="/home/airflow/composer_kube_config",
# Identifier of connection that should be used
kubernetes_conn_id="kubernetes_default",
)
Expand All @@ -135,9 +135,11 @@
# The secrets to pass to Pod, the Pod will fail to create if the
# secrets you specify in a Secret object do not exist in Kubernetes.
secrets=[secret_env, secret_volume],
# Entrypoint of the container, if not specified the Docker container's
# entrypoint is used. The cmds parameter is templated.
cmds=["echo"],
# env_vars allows you to specify environment variables for your
# container to use. env_vars is templated.
# container to use. The env_vars parameter is templated.
env_vars={
"EXAMPLE_VAR": "/example/value",
"GOOGLE_APPLICATION_CREDENTIALS": "/var/secrets/google/service-account.json",
Expand All @@ -157,7 +159,7 @@
# Entrypoint of the container, if not specified the Docker container's
# entrypoint is used. The cmds parameter is templated.
cmds=["perl"],
# Arguments to the entrypoint. The docker image's CMD is used if this
# Arguments to the entrypoint. The Docker image's CMD is used if this
# is not provided. The arguments parameter is templated.
arguments=["-Mbignum=bpi", "-wle", "print bpi(2000)"],
# The secrets to pass to Pod, the Pod will fail to create if the
Expand All @@ -167,8 +169,8 @@
labels={"pod-label": "label-name"},
# Timeout to start up the Pod, default is 600.
startup_timeout_seconds=600,
# The environment variables to be initialized in the container
# env_vars are templated.
# The environment variables to be initialized in the container.
# The env_vars parameter is templated.
env_vars={"EXAMPLE_VAR": "/example/value"},
# If true, logs stdout output of container. Defaults to True.
get_logs=True,
Expand Down