feat: video recording with pluggable upload container

Video/Dockerfile

@@ -18,10 +18,10 @@ ENV DEBIAN_FRONTEND=noninteractive \
 RUN apt-get -qqy update \
   && apt-get upgrade -yq \
   && apt-get -qqy --no-install-recommends install \
-    supervisor x11-xserver-utils python3-pip \
+    supervisor x11-xserver-utils x11-utils curl jq python3-pip \
   && python3 -m pip install --upgrade pip \
   && python3 -m pip install --upgrade setuptools \
-  && python3 -m pip install --upgrade wheel \    
+  && python3 -m pip install --upgrade wheel \
   && rm -rf /var/lib/apt/lists/* /var/cache/apt/*
 
 #======================================

charts/selenium-grid/templates/_helpers.tpl

@@ -87,6 +87,13 @@ Service Account fullname
 {{- .Values.serviceAccount.name | default "selenium-serviceaccount" | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
 
+{{/*
+Video ConfigMap fullname
+*/}}
+{{- define "seleniumGrid.video.fullname" -}}
+{{- default "selenium-video" .Values.videoRecorder.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
 {{/*
 Is autoscaling using KEDA enabled
 */}}
@@ -168,6 +175,74 @@ template:
     {{- if .node.sidecars }}
       {{- toYaml .node.sidecars | nindent 6 }}
     {{- end }}
+    {{- if .Values.videoRecorder.enabled }}
+      - name: video
+        image: {{ printf "%s:%s" .Values.videoRecorder.imageName .Values.videoRecorder.imageTag }}
+        imagePullPolicy: {{ .Values.videoRecorder.imagePullPolicy }}
+        env:
+        - name: UPLOAD_DESTINATION_PREFIX
+          value: {{ .Values.videoRecorder.uploadDestinationPrefix }}
+      {{- with .Values.videoRecorder.extraEnvironmentVariables }}
+        {{- tpl (toYaml .) $ | nindent 8 }}
+      {{- end }}
+        envFrom:
+        - configMapRef:
+            name: {{ .Values.busConfigMap.name }}
+      {{- with .Values.videoRecorder.extraEnvFrom }}
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- if gt (len .Values.videoRecorder.ports) 0 }}
+        ports:
+      {{- range .Values.videoRecorder.ports }}
+        - containerPort: {{ . }}
+          protocol: TCP
+      {{- end }}
+      {{- end }}
+        volumeMounts:
+        - name: dshm
+          mountPath: /dev/shm
+        - name: video-scripts
+          mountPath: /opt/bin/video.sh
+          subPath: video.sh
+        - name: video
+          mountPath: /videos
+      {{- if .Values.videoRecorder.extraVolumeMounts }}
+        {{- toYaml .Values.videoRecorder.extraVolumeMounts | nindent 8 }}
+      {{- end }}
+      {{- with .Values.videoRecorder.resources }}
+        resources: {{- toYaml . | nindent 10 }}
+      {{- end }}
+    {{- if .uploader }}
+      - name: uploader
+        image: {{ printf "%s:%s" .uploader.imageName .uploader.imageTag }}
+        imagePullPolicy: {{ .uploader.imagePullPolicy }}
+      {{- with .uploader.command }}
+        command: {{- tpl (toYaml .) $ | nindent 8 }}
+      {{- end }}
+      {{- with .uploader.args }}
+        args: {{- tpl (toYaml .) $ | nindent 8 }}
+      {{- end }}
+      {{- with .uploader.extraEnvironmentVariables }}
+        env: {{- tpl (toYaml .) $ | nindent 8 }}
+      {{- end }}
+        {{- with .uploader.extraEnvFrom }}
+        envFrom:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+        volumeMounts:
+        - name: video
+          mountPath: /videos
+      {{- if .uploader.extraVolumeMounts }}
+        {{- toYaml .uploader.extraVolumeMounts | nindent 8 }}
+      {{- end }}
+      {{- with .uploader.resources }}
+        resources: {{- toYaml . | nindent 10 }}
+      {{- end }}
+      {{- with .uploader.securityContext }}
+        securityContext: {{- toYaml . | nindent 10 }}
+      {{- end }}
+    {{- end }}
+    {{- end }}
   {{- if or .Values.global.seleniumGrid.imagePullSecret .node.imagePullSecret }}
     imagePullSecrets:
       - name: {{ default .Values.global.seleniumGrid.imagePullSecret .node.imagePullSecret }}
@@ -191,6 +266,14 @@ template:
     {{- if .node.extraVolumes }}
       {{ toYaml .node.extraVolumes | nindent 6 }}
     {{- end }}
+    {{- if .Values.videoRecorder.enabled }}
+      - name: video-scripts
+        configMap:
+          name: {{ template "seleniumGrid.video.fullname" . }}
+          defaultMode: 0500
+      - name: video
+      {{- toYaml .Values.videoRecorder.volume | nindent 8 }}
+  {{- end }}
 {{- end -}}
 
 {{/*

charts/selenium-grid/templates/chrome-node-deployment.yaml

@@ -25,5 +25,6 @@ spec:
 {{- $podScope := deepCopy . -}}
 {{- $_ := set $podScope "name" "selenium-chrome-node" -}}
 {{- $_ =  set $podScope "node" .Values.chromeNode  -}}
+{{- $_ =  set $podScope "uploader" (get .Values.videoRecorder .Values.videoRecorder.uploader) -}}
 {{- include "seleniumGrid.podTemplate" $podScope | nindent 2 }}
 {{- end }}

charts/selenium-grid/templates/chrome-node-scaledjobs.yaml

@@ -35,5 +35,6 @@ spec:
 {{- $podScope := deepCopy . -}}
 {{- $_ := set $podScope "name" "selenium-chrome-node" -}}
 {{- $_ =  set $podScope "node" .Values.chromeNode  -}}
+{{- $_ =  set $podScope "uploader" (get .Values.videoRecorder .Values.videoRecorder.uploader) -}}
 {{- include "seleniumGrid.podTemplate" $podScope | nindent 4 }}
 {{- end }}

charts/selenium-grid/templates/edge-node-deployment.yaml

@@ -25,5 +25,6 @@ spec:
 {{- $podScope := deepCopy . -}}
 {{- $_ := set $podScope "name" "selenium-edge-node" -}}
 {{- $_ =  set $podScope "node" .Values.edgeNode  -}}
+{{- $_ =  set $podScope "uploader" (get .Values.videoRecorder .Values.videoRecorder.uploader) -}}
 {{- include "seleniumGrid.podTemplate" $podScope | nindent 2 }}
 {{- end }}

charts/selenium-grid/templates/edge-node-scaledjob.yaml

@@ -35,5 +35,6 @@ spec:
 {{- $podScope := deepCopy . -}}
 {{- $_ := set $podScope "name" "selenium-edge-node" -}}
 {{- $_ =  set $podScope "node" .Values.edgeNode  -}}
+{{- $_ =  set $podScope "uploader" (get .Values.videoRecorder .Values.videoRecorder.uploader) -}}
 {{- include "seleniumGrid.podTemplate" $podScope | nindent 4 }}
 {{- end }}

charts/selenium-grid/templates/firefox-node-deployment.yaml

@@ -25,5 +25,6 @@ spec:
 {{- $podScope := deepCopy . -}}
 {{- $_ := set $podScope "name" "selenium-firefox-node" -}}
 {{- $_ =  set $podScope "node" .Values.firefoxNode  -}}
+{{- $_ =  set $podScope "uploader" (get .Values.videoRecorder .Values.videoRecorder.uploader) -}}
 {{- include "seleniumGrid.podTemplate" $podScope | nindent 2 }}
 {{- end }}

charts/selenium-grid/templates/firefox-node-scaledjob.yaml

@@ -35,5 +35,6 @@ spec:
 {{- $podScope := deepCopy . -}}
 {{- $_ := set $podScope "name" "selenium-firefox-node" -}}
 {{- $_ =  set $podScope "node" .Values.firefoxNode  -}}
+{{- $_ =  set $podScope "uploader" (get .Values.videoRecorder .Values.videoRecorder.uploader) -}}
 {{- include "seleniumGrid.podTemplate" $podScope | nindent 4 }}
 {{- end }}

charts/selenium-grid/templates/video-cm.yaml

@@ -0,0 +1,92 @@
+{{- if .Values.videoRecorder.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "seleniumGrid.video.fullname" . }}
+data:
+  video.sh: |
+    #!/usr/bin/env bash
+    set -em
+    function finish {
+        echo exit > /videos/uploadpipe
+        kill -s SIGINT `cat /var/run/supervisor/supervisord.pid`
+    }
+    trap finish EXIT
+    FRAME_RATE=${FRAME_RATE:-$SE_FRAME_RATE}
+    CODEC=${CODEC:-$SE_CODEC}
+    PRESET=${PRESET:-$SE_PRESET}
+    export DISPLAY=localhost:${DISPLAY_NUM}.0
+
+    return_code=1
+    max_attempts=600
+    attempts=0
+    mkfifo /videos/uploadpipe
+    if [[ "$UPLOAD_DESTINATION_PREFIX" = "" ]]
+    then
+        echo Upload destination not known since UPLOAD_DESTINATION_PREFIX is not set. Exiting video recorder.
+        exit
+    fi
+    echo Checking if the display is open
+    until xset b off || [[ $attempts = $max_attempts ]]
+    do
+        echo Waiting before next display check
+        sleep 0.5
+        attempts=$((attempts+1))
+    done
+    if [[ $attempts = $max_attempts ]]
+    then
+        echo Can not open display, exiting.
+        exit
+    fi
+    VIDEO_SIZE=$(xdpyinfo | grep 'dimensions:' | awk '{print $2}')
+
+    recording_started="false"
+    video_file_name=""
+    video_file=""
+    prev_session_id=""
+    attempts=0
+    echo Checking if node API responds
+    until curl -s --request GET http://localhost:5555/status || [[ $attempts = $max_attempts ]]
+    do
+        echo Waiting before next API check
+        sleep 0.5
+        attempts=$((attempts+1))
+    done
+    if [[ $attempts = $max_attempts ]]
+    then
+        echo Can not reach node API, exiting.
+        exit
+    fi
+    while curl -s --request GET http://localhost:5555/status > /tmp/status.json
+    do
+        session_id=$(jq -r '.[]?.node?.slots | .[0]?.session?.sessionId' /tmp/status.json)
+        echo $session_id
+        if [[ "$session_id" != "null" && "$session_id" != "" && "$recording_started" = "false" ]]
+        then
+            video_file_name="$session_id.mp4"
+            video_file="${VIDEO_LOCATION:-/videos}/$video_file_name"
+            echo "Starting to record video"
+            ffmpeg -nostdin -y -f x11grab -video_size ${VIDEO_SIZE} -r ${FRAME_RATE} -i ${DISPLAY} -codec:v ${CODEC} ${PRESET} -pix_fmt yuv420p $video_file &
+            recording_started="true"
+            echo "Video recording started"
+        elif [[ "$session_id" != "$prev_session_id" && "$recording_started" = "true" ]]
+        then
+            echo "Stopping to record video"
+            kill -INT %1
+            fg || echo ffmpeg exited with code $?
+            upload_destination=${UPLOAD_DESTINATION_PREFIX}${video_file_name}
+            echo "Uploading video to $upload_destination"
+            echo $video_file $upload_destination > /videos/uploadpipe &
+            recording_started="false"
+        elif [[ $recording_started = "true" ]]
+        then
+            echo "Video recording in progress"
+            sleep 1
+        else
+            echo "No session in progress"
+            sleep 1
+        fi
+        prev_session_id=$session_id
+    done
+    echo
+{{- end }}

charts/selenium-grid/values.yaml

@@ -24,6 +24,7 @@ serviceAccount:
   create: true
   name: ""
   annotations: {}
+  #  eks.amazonaws.com/role-arn: "arn:aws:iam::12345678:role/video-bucket-permissions"
 
 # Configure the ingress resource to access the Grid installation.
 ingress:
@@ -724,5 +725,79 @@ edgeNode:
   # It should be set using the --set-json option
   sidecars: []
 
+videoRecorder:
+  # Image of video recorder
+  imageName: selenium/video
+  enabled: false
+  # Where to upload the video file. Should be set to something like 's3://myvideobucket/'
+  uploadDestinationPrefix: ""
+  # What uploader to use. See .videRecorder.s3 for how to create a new one.
+  uploader: s3
+
+  # Image of video recorder
+  imageTag: latest
+  # Image pull policy (see https://kubernetes.io/docs/concepts/containers/images/#updating-images)
+  imagePullPolicy: IfNotPresent
+
+  ports:
+    - 5666
+  resources:
+    requests:
+      memory: "1Gi"
+      cpu: "1"
+    limits:
+      memory: "1Gi"
+      cpu: "1"
+  extraEnvironmentVariables: []
+  # - name: VIDEO_LOCATION
+  #   value: /videos
+  # Custom environment variables by sourcing entire configMap, Secret, etc. for video recorder.
+  extraEnvFrom:
+    # - configMapRef:
+    #   name: proxy-settings
+    # - secretRef:
+    #   name: mysecret
+  # Wait for pod startup
+  terminationGracePeriodSeconds: 30
+  volume:
+    emptyDir: {}
+  s3:
+    imageName: public.ecr.aws/bitnami/aws-cli
+    imageTag: "2"
+    imagePullPolicy: IfNotPresent
+    securityContext:
+      runAsUser: 0
+    command:
+    - /bin/sh
+    args:
+    - -c
+    - |
+      while ! [ -p /videos/uploadpipe ]
+      do
+          echo Waiting for /videos/uploadpipe to be created
+          sleep 1
+      done
+      echo Waiting for files to upload
+      while read FILE DESTINATION < /videos/uploadpipe
+      do
+          if [ "$FILE" = "exit" ]
+          then
+              break
+          else
+              aws s3 cp --no-progress $FILE $DESTINATION
+          fi
+      done
+  # extraEnvironmentVariables:
+  # - name: AWS_ACCESS_KEY_ID
+  #   value: aws_access_key_id
+  # - name: AWS_SECRET_ACCESS_KEY
+  #   value: aws_secret_access_key
+  # - name:
+  #   valueFrom:
+  #     secretKeyRef:
+  #       name: secret-name
+  #       key: secret-key
+
+
 # Custom labels for k8s resources
 customLabels: {}