Skip to content
This repository was archived by the owner on Nov 16, 2023. It is now read-only.

Commit 3420ae0

Browse files
yqwang-msyqwang-ms
authored
[BREAKING CHANGE]: Refine AnnotationKey, LabelKey and EnvName (#6)
1. Change "POD_NAMESPACE" to "FRAMEWORK_NAMESPACE" 2. Prefix "FC_" for all FrameworkController Predefined AnnotationKeys, LabelKeys and EnvNames 3. Prefix "FB_" and uppercase TaskRoleName for all FrameworkBarrier EnvNames
1 parent c06876a commit 3420ae0

File tree

9 files changed

+105
-100
lines changed

9 files changed

+105
-100
lines changed

‎example/framework/basic/batchstatefulfailed.yaml‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -27,18 +27,19 @@ spec:
2727
# To locate a specific Task during its whole lifecycle regardless of
2828
# any retry:
2929
# Consistent Identity:
30-
# PodName = {FrameworkName}-{TaskRoleName}-{TaskIndex}
3130
# PodNamespace = {FrameworkNamespace}
31+
# PodName = {FrameworkName}-{TaskRoleName}-{TaskIndex}
3232
# Consistent Environment Variable Value:
33-
# ${FRAMEWORK_NAME}, ${TASKROLE_NAME}, ${TASK_INDEX}
34-
# ${CONFIGMAP_NAME}, ${POD_NAME}, ${POD_NAMESPACE}
33+
# ${FC_FRAMEWORK_NAMESPACE},
34+
# ${FC_FRAMEWORK_NAME}, ${FC_TASKROLE_NAME}, ${FC_TASK_INDEX},
35+
# ${FC_CONFIGMAP_NAME}, ${FC_POD_NAME}
3536
#
3637
# To locate a specific execution attempt of a specific Task:
3738
# Attempt Specific Environment Variable Value:
38-
# ${FRAMEWORK_ATTEMPT_ID}, ${TASK_ATTEMPT_ID}
39+
# ${FC_FRAMEWORK_ATTEMPT_ID}, ${FC_TASK_ATTEMPT_ID}
3940
#
4041
# To locate a specific execution attempt instance of a specific Task:
4142
# Attempt Instance Specific Environment Variable Value:
42-
# ${FRAMEWORK_ATTEMPT_INSTANCE_UID}, ${CONFIGMAP_UID}
43-
# ${TASK_ATTEMPT_INSTANCE_UID}, ${POD_UID}
43+
# ${FC_FRAMEWORK_ATTEMPT_INSTANCE_UID}, ${FC_CONFIGMAP_UID}
44+
# ${FC_TASK_ATTEMPT_INSTANCE_UID}, ${FC_POD_UID}
4445
command: ["sh", "-c", "printenv && sleep 60 && exit 1"]

‎example/framework/basic/service.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,8 @@ metadata:
3838
spec:
3939
selector:
4040
# Using predefined labels
41-
FRAMEWORK_NAME: service
42-
TASKROLE_NAME: server
41+
FC_FRAMEWORK_NAME: service
42+
FC_TASKROLE_NAME: server
4343
# Also can use customized labels
4444
#app: server
4545
ports:

‎example/framework/basic/servicestateful.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,8 @@ metadata:
4343
spec:
4444
selector:
4545
# See comments in service.yaml
46-
FRAMEWORK_NAME: servicestateful
47-
TASKROLE_NAME: serverstateful
46+
FC_FRAMEWORK_NAME: servicestateful
47+
FC_TASKROLE_NAME: serverstateful
4848
ports:
4949
- port: 80
5050
type: NodePort

‎example/framework/extension/frameworkbarrier.yaml‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -29,15 +29,15 @@ spec:
2929
containers:
3030
- name: ubuntu
3131
image: ubuntu:trusty
32-
# Using /mnt/frameworkbarrier/injector.sh to inject environment
33-
# variables, such as:
34-
# {TaskRoleName}_ips=
32+
# Using /mnt/frameworkbarrier/injector.sh to inject environment variables,
33+
# such as:
34+
# FB_{UpperCase({TaskRoleName})}_IPS=
3535
# {Task[0].PodIP},...,
3636
# {Task[TaskRole.TaskNumber-1].PodIP}
37-
# {TaskRoleName}_addresses=
38-
# {Task[0].PodIP}:${{TaskRoleName}_port},...,
39-
# {Task[TaskRole.TaskNumber-1].PodIP}:${{TaskRoleName}_port}
40-
# Note, the environment variable {TaskRoleName}_port should be
37+
# FB_{UpperCase({TaskRoleName})}_ADDRESSES=
38+
# {Task[0].PodIP}:${FB_{UpperCase({TaskRoleName})}_PORT},...,
39+
# {Task[TaskRole.TaskNumber-1].PodIP}:${FB_{UpperCase({TaskRoleName})}_PORT}
40+
# Note, the environment variable FB_{UpperCase({TaskRoleName})}_PORT should be
4141
# provided by the caller in advance.
4242
#
4343
# User may need to tweak these environment variables to its own
@@ -48,8 +48,8 @@ spec:
4848
# /mnt/frameworkbarrier/framework.json.
4949
command: [
5050
"sh", "-c",
51-
"server_port=4001 worker_port=5001 . /mnt/frameworkbarrier/injector.sh && printenv &&
52-
server_port=4002 worker_port=5002 . /mnt/frameworkbarrier/injector.sh && printenv &&
51+
"FB_SERVER_PORT=4001 FB_WORKER_PORT=5001 . /mnt/frameworkbarrier/injector.sh && printenv &&
52+
FB_SERVER_PORT=4002 FB_WORKER_PORT=5002 . /mnt/frameworkbarrier/injector.sh && printenv &&
5353
sleep 60"]
5454
ports:
5555
- containerPort: 4001
@@ -104,8 +104,8 @@ spec:
104104
image: ubuntu:trusty
105105
command: [
106106
"sh", "-c",
107-
"server_port=4001 worker_port=5001 . /mnt/frameworkbarrier/injector.sh && printenv &&
108-
server_port=4002 worker_port=5002 . /mnt/frameworkbarrier/injector.sh && printenv &&
107+
"FB_SERVER_PORT=4001 FB_WORKER_PORT=5001 . /mnt/frameworkbarrier/injector.sh && printenv &&
108+
FB_SERVER_PORT=4002 FB_WORKER_PORT=5002 . /mnt/frameworkbarrier/injector.sh && printenv &&
109109
sleep 60"]
110110
ports:
111111
- containerPort: 5001

‎example/framework/scenario/tensorflow/cpu/tensorflowdistributedtrainingwithcpu.yaml‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -43,22 +43,22 @@ spec:
4343
# For the tf_cnn_benchmarks usage, see
4444
# https://github.com/tensorflow/benchmarks/tree/master/scripts/tf_cnn_benchmarks
4545
workingDir: /tensorflow/benchmarks/scripts/tf_cnn_benchmarks
46-
# Using /mnt/frameworkbarrier/injector.sh to inject environment
47-
# variables without the need for image invasion and k8s DNS:
48-
# {TaskRoleName}_addresses=
49-
# {Task[0].PodIP}:${{TaskRoleName}_port},...,
50-
# {Task[TaskRole.TaskNumber-1].PodIP}:${{TaskRoleName}_port}
46+
# Using /mnt/frameworkbarrier/injector.sh to inject environment variables
47+
# without the need for image invasion and k8s DNS:
48+
# FB_{UpperCase({TaskRoleName})}_ADDRESSES=
49+
# {Task[0].PodIP}:${FB_{UpperCase({TaskRoleName})}_PORT},...,
50+
# {Task[TaskRole.TaskNumber-1].PodIP}:${FB_{UpperCase({TaskRoleName})}_PORT}
5151
# See more in ./example/framework/extension/frameworkbarrier.yaml
5252
command: [
5353
"sh", "-c",
54-
"ps_port=4001 worker_port=5001 . /mnt/frameworkbarrier/injector.sh &&
55-
python tf_cnn_benchmarks.py --job_name=ps --task_index=${TASK_INDEX}
56-
--ps_hosts=${ps_addresses} --worker_hosts=${worker_addresses}
54+
"FB_PS_PORT=4001 FB_WORKER_PORT=5001 . /mnt/frameworkbarrier/injector.sh &&
55+
python tf_cnn_benchmarks.py --job_name=ps --task_index=${FC_TASK_INDEX}
56+
--ps_hosts=${FB_PS_ADDRESSES} --worker_hosts=${FB_WORKER_ADDRESSES}
5757
--variable_update=parameter_server --cross_replica_sync=false
5858
--model=alexnet --batch_size=8 --num_batches=10
5959
--device=cpu --local_parameter_device=cpu --data_format=NHWC
6060
--data_name=cifar10 --data_dir=/mnt/data/cifar-10-batches-py
61-
--train_dir=/mnt/data/${FRAMEWORK_NAME}/output"]
61+
--train_dir=/mnt/data/${FC_FRAMEWORK_NAME}/output"]
6262
ports:
6363
- containerPort: 4001
6464
volumeMounts:
@@ -129,14 +129,14 @@ spec:
129129
workingDir: /tensorflow/benchmarks/scripts/tf_cnn_benchmarks
130130
command: [
131131
"sh", "-c",
132-
"ps_port=4001 worker_port=5001 . /mnt/frameworkbarrier/injector.sh &&
133-
python tf_cnn_benchmarks.py --job_name=worker --task_index=${TASK_INDEX}
134-
--ps_hosts=${ps_addresses} --worker_hosts=${worker_addresses}
132+
"FB_PS_PORT=4001 FB_WORKER_PORT=5001 . /mnt/frameworkbarrier/injector.sh &&
133+
python tf_cnn_benchmarks.py --job_name=worker --task_index=${FC_TASK_INDEX}
134+
--ps_hosts=${FB_PS_ADDRESSES} --worker_hosts=${FB_WORKER_ADDRESSES}
135135
--variable_update=parameter_server --cross_replica_sync=false
136136
--model=alexnet --batch_size=8 --num_batches=10
137137
--device=cpu --local_parameter_device=cpu --data_format=NHWC
138138
--data_name=cifar10 --data_dir=/mnt/data/cifar-10-batches-py
139-
--train_dir=/mnt/data/${FRAMEWORK_NAME}/output"]
139+
--train_dir=/mnt/data/${FC_FRAMEWORK_NAME}/output"]
140140
ports:
141141
- containerPort: 5001
142142
volumeMounts:

‎example/framework/scenario/tensorflow/gpu/tensorflowdistributedtrainingwithgpu.yaml‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -43,22 +43,22 @@ spec:
4343
# For the tf_cnn_benchmarks usage, see
4444
# https://github.com/tensorflow/benchmarks/tree/master/scripts/tf_cnn_benchmarks
4545
workingDir: /tensorflow/benchmarks/scripts/tf_cnn_benchmarks
46-
# Using /mnt/frameworkbarrier/injector.sh to inject environment
47-
# variables without the need for image invasion and k8s DNS:
48-
# {TaskRoleName}_addresses=
49-
# {Task[0].PodIP}:${{TaskRoleName}_port},...,
50-
# {Task[TaskRole.TaskNumber-1].PodIP}:${{TaskRoleName}_port}
46+
# Using /mnt/frameworkbarrier/injector.sh to inject environment variables
47+
# without the need for image invasion and k8s DNS:
48+
# FB_{UpperCase({TaskRoleName})}_ADDRESSES=
49+
# {Task[0].PodIP}:${FB_{UpperCase({TaskRoleName})}_PORT},...,
50+
# {Task[TaskRole.TaskNumber-1].PodIP}:${FB_{UpperCase({TaskRoleName})}_PORT}
5151
# See more in ./example/framework/extension/frameworkbarrier.yaml
5252
command: [
5353
"sh", "-c",
54-
"ps_port=4001 worker_port=5001 . /mnt/frameworkbarrier/injector.sh &&
55-
python tf_cnn_benchmarks.py --job_name=ps --task_index=${TASK_INDEX}
56-
--ps_hosts=${ps_addresses} --worker_hosts=${worker_addresses}
54+
"FB_PS_PORT=4001 FB_WORKER_PORT=5001 . /mnt/frameworkbarrier/injector.sh &&
55+
python tf_cnn_benchmarks.py --job_name=ps --task_index=${FC_TASK_INDEX}
56+
--ps_hosts=${FB_PS_ADDRESSES} --worker_hosts=${FB_WORKER_ADDRESSES}
5757
--variable_update=parameter_server --cross_replica_sync=false
5858
--model=alexnet --batch_size=8 --num_batches=10
5959
--device=gpu --local_parameter_device=gpu --num_gpus=1 --data_format=NCHW
6060
--data_name=cifar10 --data_dir=/mnt/data/cifar-10-batches-py
61-
--train_dir=/mnt/data/${FRAMEWORK_NAME}/output"]
61+
--train_dir=/mnt/data/${FC_FRAMEWORK_NAME}/output"]
6262
ports:
6363
- containerPort: 4001
6464
resources:
@@ -135,14 +135,14 @@ spec:
135135
workingDir: /tensorflow/benchmarks/scripts/tf_cnn_benchmarks
136136
command: [
137137
"sh", "-c",
138-
"ps_port=4001 worker_port=5001 . /mnt/frameworkbarrier/injector.sh &&
139-
python tf_cnn_benchmarks.py --job_name=worker --task_index=${TASK_INDEX}
140-
--ps_hosts=${ps_addresses} --worker_hosts=${worker_addresses}
138+
"FB_PS_PORT=4001 FB_WORKER_PORT=5001 . /mnt/frameworkbarrier/injector.sh &&
139+
python tf_cnn_benchmarks.py --job_name=worker --task_index=${FC_TASK_INDEX}
140+
--ps_hosts=${FB_PS_ADDRESSES} --worker_hosts=${FB_WORKER_ADDRESSES}
141141
--variable_update=parameter_server --cross_replica_sync=false
142142
--model=alexnet --batch_size=8 --num_batches=10
143143
--device=gpu --local_parameter_device=gpu --num_gpus=1 --data_format=NCHW
144144
--data_name=cifar10 --data_dir=/mnt/data/cifar-10-batches-py
145-
--train_dir=/mnt/data/${FRAMEWORK_NAME}/output"]
145+
--train_dir=/mnt/data/${FC_FRAMEWORK_NAME}/output"]
146146
ports:
147147
- containerPort: 5001
148148
resources:

‎pkg/apis/frameworkcontroller/v1/constants.go‎

Lines changed: 25 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -46,35 +46,40 @@ const (
4646
ExtendedUnlimitedValue = -2
4747

4848
// For all managed objects
49-
AnnotationKeyFrameworkName = "FRAMEWORK_NAME"
50-
AnnotationKeyTaskRoleName = "TASKROLE_NAME"
51-
AnnotationKeyTaskIndex = "TASK_INDEX"
52-
AnnotationKeyConfigMapName = "CONFIGMAP_NAME"
53-
AnnotationKeyPodName = "POD_NAME"
54-
AnnotationKeyPodNamespace = "POD_NAMESPACE"
55-
56-
AnnotationKeyFrameworkAttemptID = "FRAMEWORK_ATTEMPT_ID"
57-
AnnotationKeyFrameworkAttemptInstanceUID = "FRAMEWORK_ATTEMPT_INSTANCE_UID"
58-
AnnotationKeyConfigMapUID = "CONFIGMAP_UID"
59-
AnnotationKeyTaskAttemptID = "TASK_ATTEMPT_ID"
60-
49+
// Predefined Annotations
50+
AnnotationKeyFrameworkNamespace = "FC_FRAMEWORK_NAMESPACE"
51+
AnnotationKeyFrameworkName = "FC_FRAMEWORK_NAME"
52+
AnnotationKeyTaskRoleName = "FC_TASKROLE_NAME"
53+
AnnotationKeyTaskIndex = "FC_TASK_INDEX"
54+
AnnotationKeyConfigMapName = "FC_CONFIGMAP_NAME"
55+
AnnotationKeyPodName = "FC_POD_NAME"
56+
57+
AnnotationKeyFrameworkAttemptID = "FC_FRAMEWORK_ATTEMPT_ID"
58+
AnnotationKeyFrameworkAttemptInstanceUID = "FC_FRAMEWORK_ATTEMPT_INSTANCE_UID"
59+
AnnotationKeyConfigMapUID = "FC_CONFIGMAP_UID"
60+
AnnotationKeyTaskAttemptID = "FC_TASK_ATTEMPT_ID"
61+
62+
// Predefined Labels
6163
LabelKeyFrameworkName = AnnotationKeyFrameworkName
6264
LabelKeyTaskRoleName = AnnotationKeyTaskRoleName
6365

6466
// For all managed containers
65-
EnvNameFrameworkName = AnnotationKeyFrameworkName
66-
EnvNameTaskRoleName = AnnotationKeyTaskRoleName
67-
EnvNameTaskIndex = AnnotationKeyTaskIndex
68-
EnvNameConfigMapName = AnnotationKeyConfigMapName
69-
EnvNamePodName = AnnotationKeyPodName
70-
EnvNamePodNamespace = AnnotationKeyPodNamespace
67+
// Predefined Environment Variables
68+
// It can be referred by the environment variable specified in the spec, i.e.
69+
// specify the environment variable value to include "$(AnyPredefinedEnvName)".
70+
EnvNameFrameworkNamespace = AnnotationKeyFrameworkNamespace
71+
EnvNameFrameworkName = AnnotationKeyFrameworkName
72+
EnvNameTaskRoleName = AnnotationKeyTaskRoleName
73+
EnvNameTaskIndex = AnnotationKeyTaskIndex
74+
EnvNameConfigMapName = AnnotationKeyConfigMapName
75+
EnvNamePodName = AnnotationKeyPodName
7176

7277
EnvNameFrameworkAttemptID = AnnotationKeyFrameworkAttemptID
7378
EnvNameFrameworkAttemptInstanceUID = AnnotationKeyFrameworkAttemptInstanceUID
7479
EnvNameConfigMapUID = AnnotationKeyConfigMapUID
7580
EnvNameTaskAttemptID = AnnotationKeyTaskAttemptID
76-
EnvNameTaskAttemptInstanceUID = "TASK_ATTEMPT_INSTANCE_UID"
77-
EnvNamePodUID = "POD_UID"
81+
EnvNameTaskAttemptInstanceUID = "FC_TASK_ATTEMPT_INSTANCE_UID"
82+
EnvNamePodUID = "FC_POD_UID"
7883
)
7984

8085
var FrameworkGroupVersionKind = SchemeGroupVersion.WithKind(FrameworkKind)

‎pkg/apis/frameworkcontroller/v1/funcs.go‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -261,6 +261,7 @@ func (f *Framework) NewConfigMap() *core.ConfigMap {
261261
cm.Finalizers = []string{meta.FinalizerDeleteDependents}
262262

263263
cm.Annotations = map[string]string{}
264+
cm.Annotations[AnnotationKeyFrameworkNamespace] = f.Namespace
264265
cm.Annotations[AnnotationKeyFrameworkName] = f.Name
265266
cm.Annotations[AnnotationKeyConfigMapName] = cm.Name
266267
cm.Annotations[AnnotationKeyFrameworkAttemptID] = frameworkAttemptIDStr
@@ -305,12 +306,12 @@ func (f *Framework) NewPod(cm *core.ConfigMap, taskRoleName string, taskIndex in
305306
if pod.Annotations == nil {
306307
pod.Annotations = map[string]string{}
307308
}
309+
pod.Annotations[AnnotationKeyFrameworkNamespace] = f.Namespace
308310
pod.Annotations[AnnotationKeyFrameworkName] = f.Name
309311
pod.Annotations[AnnotationKeyTaskRoleName] = taskRoleName
310312
pod.Annotations[AnnotationKeyTaskIndex] = taskIndexStr
311313
pod.Annotations[AnnotationKeyConfigMapName] = f.ConfigMapName()
312314
pod.Annotations[AnnotationKeyPodName] = pod.Name
313-
pod.Annotations[AnnotationKeyPodNamespace] = pod.Namespace
314315
pod.Annotations[AnnotationKeyFrameworkAttemptID] = frameworkAttemptIDStr
315316
pod.Annotations[AnnotationKeyFrameworkAttemptInstanceUID] = frameworkAttemptInstanceUIDStr
316317
pod.Annotations[AnnotationKeyConfigMapUID] = configMapUIDStr
@@ -322,13 +323,13 @@ func (f *Framework) NewPod(cm *core.ConfigMap, taskRoleName string, taskIndex in
322323
pod.Labels[LabelKeyFrameworkName] = f.Name
323324
pod.Labels[LabelKeyTaskRoleName] = taskRoleName
324325

325-
exEnvs := []core.EnvVar{
326+
predefinedEnvs := []core.EnvVar{
327+
{Name: EnvNameFrameworkNamespace, Value: f.Namespace},
326328
{Name: EnvNameFrameworkName, Value: f.Name},
327329
{Name: EnvNameTaskRoleName, Value: taskRoleName},
328330
{Name: EnvNameTaskIndex, Value: taskIndexStr},
329331
{Name: EnvNameConfigMapName, Value: f.ConfigMapName()},
330332
{Name: EnvNamePodName, Value: pod.Name},
331-
{Name: EnvNamePodNamespace, Value: pod.Namespace},
332333
{Name: EnvNameFrameworkAttemptID, Value: frameworkAttemptIDStr},
333334
{Name: EnvNameFrameworkAttemptInstanceUID, Value: frameworkAttemptInstanceUIDStr},
334335
{Name: EnvNameConfigMapUID, Value: configMapUIDStr},
@@ -337,24 +338,22 @@ func (f *Framework) NewPod(cm *core.ConfigMap, taskRoleName string, taskIndex in
337338
{Name: EnvNameTaskAttemptInstanceUID, Value: taskAttemptInstanceUIDReferStr},
338339
}
339340

341+
// Prepend predefinedEnvs so that they can be referred by the environment variable
342+
// specified in the spec.
340343
// Change the default TerminationMessagePolicy to TerminationMessageFallbackToLogsOnError
341344
// in case the cluster-level logging has not been setup for the cluster.
342345
// See https://kubernetes.io/docs/concepts/cluster-administration/logging
343346
// It is safe to do so, since it will only fall back to the tail log if the container
344347
// is failed and the termination message file specified by the terminationMessagePath
345348
// is not found or empty.
346349
for i := range pod.Spec.Containers {
347-
for _, exEnv := range exEnvs {
348-
pod.Spec.Containers[i].Env = append(pod.Spec.Containers[i].Env, exEnv)
349-
}
350+
pod.Spec.Containers[i].Env = append(predefinedEnvs, pod.Spec.Containers[i].Env...)
350351
if len(pod.Spec.Containers[i].TerminationMessagePolicy) == 0 {
351352
pod.Spec.Containers[i].TerminationMessagePolicy = core.TerminationMessageFallbackToLogsOnError
352353
}
353354
}
354355
for i := range pod.Spec.InitContainers {
355-
for _, exEnv := range exEnvs {
356-
pod.Spec.InitContainers[i].Env = append(pod.Spec.InitContainers[i].Env, exEnv)
357-
}
356+
pod.Spec.InitContainers[i].Env = append(predefinedEnvs, pod.Spec.InitContainers[i].Env...)
358357
if len(pod.Spec.InitContainers[i].TerminationMessagePolicy) == 0 {
359358
pod.Spec.InitContainers[i].TerminationMessagePolicy = core.TerminationMessageFallbackToLogsOnError
360359
}

0 commit comments

Comments
 (0)