Skip to content

Commit 1aa020c

Browse files
committed
K8s: Add default values for multiple nodes platform and version
Signed-off-by: Viet Nguyen Duc <nguyenducviet4496@gmail.com>
1 parent 02ea1a7 commit 1aa020c

File tree

9 files changed

+249
-44
lines changed

9 files changed

+249
-44
lines changed

Makefile

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -948,15 +948,15 @@ chart_test_autoscaling_deployment:
948948
./tests/charts/make/chart_test.sh DeploymentAutoscaling
949949

950950
chart_test_autoscaling_job_https:
951-
PLATFORMS=$(PLATFORMS) TEST_EXISTING_KEDA=true RELEASE_NAME=selenium CHART_ENABLE_BASIC_AUTH=true SELENIUM_GRID_MONITORING=false TEST_PATCHED_KEDA=$(TEST_PATCHED_KEDA) \
951+
PLATFORMS=$(PLATFORMS) TEST_EXISTING_KEDA=true RELEASE_NAME=selenium CHART_ENABLE_BASIC_AUTH=true SELENIUM_GRID_MONITORING=false TEST_PATCHED_KEDA=$(TEST_PATCHED_KEDA) TEST_MULTIPLE_PLATFORMS=true \
952952
SECURE_CONNECTION_SERVER=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_PORT=443 SUB_PATH=/ \
953953
MAX_SESSIONS_FIREFOX=1 MAX_SESSIONS_EDGE=2 MAX_SESSIONS_CHROME=3 TEST_NAME_OVERRIDE=true \
954954
VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) KEDA_BASED_NAME=$(KEDA_BASED_NAME) KEDA_BASED_TAG=$(KEDA_BASED_TAG) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) BASE_VERSION=$(BASE_VERSION) EXTERNAL_UPLOADER_CONFIG=true \
955955
TEMPLATE_OUTPUT_FILENAME="k8s_prefixSelenium_basicAuth_secureServer_autoScaling_scaledJob_existingKEDA.yaml" \
956956
./tests/charts/make/chart_test.sh JobAutoscaling
957957

958958
chart_test_autoscaling_job_hostname:
959-
PLATFORMS=$(PLATFORMS) CHART_ENABLE_TRACING=true CHART_ENABLE_BASIC_AUTH=true BASIC_AUTH_EMBEDDED_URL=true TEST_PATCHED_KEDA=$(TEST_PATCHED_KEDA) \
959+
PLATFORMS=$(PLATFORMS) CHART_ENABLE_TRACING=true CHART_ENABLE_BASIC_AUTH=true BASIC_AUTH_EMBEDDED_URL=true TEST_PATCHED_KEDA=$(TEST_PATCHED_KEDA) TEST_MULTIPLE_PLATFORMS=true \
960960
SECURE_INGRESS_ONLY_DEFAULT=true SECURE_USE_EXTERNAL_CERT=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_HOST=$$(hostname -I | cut -d' ' -f1) SELENIUM_GRID_PORT=443 \
961961
VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) KEDA_BASED_NAME=$(KEDA_BASED_NAME) KEDA_BASED_TAG=$(KEDA_BASED_TAG) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) BASE_VERSION=$(BASE_VERSION) \
962962
TEMPLATE_OUTPUT_FILENAME="k8s_enableTracing_basicAuth_secureIngress_externalCerts_ingressPublicIP_autoScaling_originKEDA_scaledJob_subPath.yaml" \
@@ -977,7 +977,7 @@ chart_test_autoscaling_job:
977977
./tests/charts/make/chart_test.sh JobAutoscaling
978978

979979
chart_test_autoscaling_playwright_connect_grid:
980-
PLATFORMS=$(PLATFORMS) CHART_FULL_DISTRIBUTED_MODE=true CHART_ENABLE_BASIC_AUTH=true TEST_EXTERNAL_DATASTORE=redis MATRIX_TESTS=CDPTests TEST_PATCHED_KEDA=$(TEST_PATCHED_KEDA) \
980+
PLATFORMS=$(PLATFORMS) CHART_FULL_DISTRIBUTED_MODE=true CHART_ENABLE_BASIC_AUTH=true TEST_EXTERNAL_DATASTORE=redis MATRIX_TESTS=CDPTests TEST_PATCHED_KEDA=$(TEST_PATCHED_KEDA) TEST_MULTIPLE_PLATFORMS=true \
981981
BASIC_AUTH_USERNAME=docker-selenium BASIC_AUTH_PASSWORD=2NMI4jdBi6k7bENoeUfV25295VvzwAE9chM24a+2VL95uOHozo \
982982
SECURE_INGRESS_ONLY_DEFAULT=true SECURE_USE_EXTERNAL_CERT=true SELENIUM_GRID_PROTOCOL=https SELENIUM_GRID_HOST=$$(hostname -I | cut -d' ' -f1) SELENIUM_GRID_PORT=443 \
983983
VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) KEDA_BASED_NAME=$(KEDA_BASED_NAME) KEDA_BASED_TAG=$(KEDA_BASED_TAG) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) BASE_VERSION=$(BASE_VERSION) \
@@ -993,7 +993,7 @@ test_k8s_autoscaling_job_count_strategy_default_with_node_max_sessions:
993993
make test_k8s_autoscaling_job_count_strategy_default
994994

995995
test_k8s_autoscaling_job_count_strategy_default:
996-
MATRIX_TESTS=$(or $(MATRIX_TESTS), "AutoscalingTestsScaleUp") SCALING_STRATEGY=$(or $(SCALING_STRATEGY), "default") \
996+
MATRIX_TESTS=$(or $(MATRIX_TESTS), "AutoscalingTestsScaleUp") SCALING_STRATEGY=$(or $(SCALING_STRATEGY), "default") TEST_MULTIPLE_PLATFORMS=true \
997997
PLATFORMS=$(PLATFORMS) RELEASE_NAME=selenium TEST_PATCHED_KEDA=$(TEST_PATCHED_KEDA) SELENIUM_GRID_PROTOCOL=http SELENIUM_GRID_HOST=localhost SELENIUM_GRID_PORT=80 \
998998
SELENIUM_GRID_MONITORING=false CLEAR_POD_HISTORY=true SET_MAX_REPLICAS=100 ENABLE_VIDEO_RECORDER=false \
999999
VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) KEDA_BASED_NAME=$(KEDA_BASED_NAME) KEDA_BASED_TAG=$(KEDA_BASED_TAG) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) BASE_VERSION=$(BASE_VERSION) \
@@ -1008,7 +1008,7 @@ test_k8s_autoscaling_deployment_count_with_node_max_sessions:
10081008
make test_k8s_autoscaling_deployment_count
10091009

10101010
test_k8s_autoscaling_deployment_count:
1011-
MATRIX_TESTS=$(or $(MATRIX_TESTS), "AutoscalingTestsScaleUp") \
1011+
MATRIX_TESTS=$(or $(MATRIX_TESTS), "AutoscalingTestsScaleUp") TEST_MULTIPLE_PLATFORMS=true \
10121012
PLATFORMS=$(PLATFORMS) RELEASE_NAME=selenium TEST_PATCHED_KEDA=$(TEST_PATCHED_KEDA) SELENIUM_GRID_PROTOCOL=http SELENIUM_GRID_HOST=localhost SELENIUM_GRID_PORT=80 \
10131013
SELENIUM_GRID_MONITORING=false CLEAR_POD_HISTORY=true SET_MAX_REPLICAS=100 ENABLE_VIDEO_RECORDER=false \
10141014
VERSION=$(TAG_VERSION) VIDEO_TAG=$(FFMPEG_TAG_VERSION)-$(BUILD_DATE) KEDA_BASED_NAME=$(KEDA_BASED_NAME) KEDA_BASED_TAG=$(KEDA_BASED_TAG) NAMESPACE=$(NAMESPACE) BINDING_VERSION=$(BINDING_VERSION) BASE_VERSION=$(BASE_VERSION) \

charts/selenium-grid/CONFIGURATION.md

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -423,10 +423,10 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
423423
| chromeNode.scaledOptions | string | `nil` | Override the scaled options for chrome nodes |
424424
| chromeNode.scaledJobOptions | string | `nil` | Override the scaledJobOptions for chrome nodes |
425425
| chromeNode.scaledObjectOptions | string | `nil` | Override the scaledObjectOptions for chrome nodes |
426-
| chromeNode.hpa.browserName | string | `"chrome"` | browserName from the capability |
426+
| chromeNode.hpa.browserName | string | `"chrome"` | browserName should match with Node stereotype and request capability is scaled by this scaler |
427427
| chromeNode.hpa.sessionBrowserName | string | `"chrome"` | sessionBrowserName if the browserName is different from the sessionBrowserName |
428-
| chromeNode.hpa.browserVersion | string | `""` | browserVersion from the capability |
429-
| chromeNode.hpa.platformName | string | `"Linux"` | platformName from the capability |
428+
| chromeNode.hpa.browserVersion | string | `""` | browserVersion should match with Node stereotype and request capability is scaled by this scaler |
429+
| chromeNode.hpa.platformName | string | `""` | platformName should match with Node stereotype and request capability is scaled by this scaler |
430430
| chromeNode.hpa.unsafeSsl | string | `"{{ template \"seleniumGrid.graphqlURL.unsafeSsl\" . }}"` | Skip check SSL when connecting to the Graphql endpoint |
431431
| chromeNode.initContainers | list | `[]` | It is used to add initContainers in the same pod of the browser node. It should be set using the --set-json option |
432432
| chromeNode.sidecars | list | `[]` | It is used to add sidecars proxy in the same pod of the browser node. It means it will add a new container to the deployment itself. It should be set using the --set-json option |
@@ -475,10 +475,10 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
475475
| firefoxNode.scaledOptions | string | `nil` | Override the scaled options for firefox nodes |
476476
| firefoxNode.scaledJobOptions | string | `nil` | Override the scaledJobOptions for firefox nodes |
477477
| firefoxNode.scaledObjectOptions | string | `nil` | Override the scaledObjectOptions for firefox nodes |
478-
| firefoxNode.hpa.browserName | string | `"firefox"` | browserName from the capability |
478+
| firefoxNode.hpa.browserName | string | `"firefox"` | browserName should match with Node stereotype and request capability is scaled by this scaler |
479479
| firefoxNode.hpa.sessionBrowserName | string | `"firefox"` | sessionBrowserName if the browserName is different from the sessionBrowserName |
480-
| firefoxNode.hpa.browserVersion | string | `""` | browserVersion from the capability |
481-
| firefoxNode.hpa.platformName | string | `"Linux"` | platformName from the capability |
480+
| firefoxNode.hpa.browserVersion | string | `""` | browserVersion should match with Node stereotype and request capability is scaled by this scaler |
481+
| firefoxNode.hpa.platformName | string | `""` | platformName should match with Node stereotype and request capability is scaled by this scaler |
482482
| firefoxNode.hpa.unsafeSsl | string | `"{{ template \"seleniumGrid.graphqlURL.unsafeSsl\" . }}"` | Skip check SSL when connecting to the Graphql endpoint |
483483
| firefoxNode.initContainers | list | `[]` | It is used to add initContainers in the same pod of the browser node. It should be set using the --set-json option |
484484
| firefoxNode.sidecars | list | `[]` | It is used to add sidecars proxy in the same pod of the browser node. It means it will add a new container to the deployment itself. It should be set using the --set-json option |
@@ -527,10 +527,10 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
527527
| edgeNode.scaledOptions | string | `nil` | Override the scaled options for edge nodes |
528528
| edgeNode.scaledJobOptions | string | `nil` | Override the scaledJobOptions for edge nodes |
529529
| edgeNode.scaledObjectOptions | string | `nil` | Override the scaledObjectOptions for edge nodes |
530-
| edgeNode.hpa.browserName | string | `"MicrosoftEdge"` | browserName from the capability |
530+
| edgeNode.hpa.browserName | string | `"MicrosoftEdge"` | browserName should match with Node stereotype and request capability is scaled by this scaler |
531531
| edgeNode.hpa.sessionBrowserName | string | `"msedge"` | sessionBrowserName if the browserName is different from the sessionBrowserName |
532-
| edgeNode.hpa.browserVersion | string | `""` | browserVersion from the capability |
533-
| edgeNode.hpa.platformName | string | `"Linux"` | platformName from the capability |
532+
| edgeNode.hpa.browserVersion | string | `""` | browserVersion should match with Node stereotype and request capability is scaled by this scaler |
533+
| edgeNode.hpa.platformName | string | `""` | platformName should match with Node stereotype and request capability is scaled by this scaler |
534534
| edgeNode.hpa.unsafeSsl | string | `"{{ template \"seleniumGrid.graphqlURL.unsafeSsl\" . }}"` | Skip check SSL when connecting to the Graphql endpoint |
535535
| edgeNode.initContainers | list | `[]` | It is used to add initContainers in the same pod of the browser node. It should be set using the --set-json option |
536536
| edgeNode.sidecars | list | `[]` | It is used to add sidecars proxy in the same pod of the browser node. It means it will add a new container to the deployment itself. It should be set using the --set-json option |
@@ -579,10 +579,10 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
579579
| relayNode.scaledOptions | string | `nil` | Override the scaled options for relay nodes |
580580
| relayNode.scaledJobOptions | string | `nil` | Override the scaledJobOptions for relay nodes |
581581
| relayNode.scaledObjectOptions | string | `nil` | Override the scaledObjectOptions for relay nodes |
582-
| relayNode.hpa.browserName | string | `"chrome"` | browserName from the capability |
582+
| relayNode.hpa.browserName | string | `"chrome"` | browserName should match with Node stereotype and request capability is scaled by this scaler |
583583
| relayNode.hpa.sessionBrowserName | string | `""` | sessionBrowserName if the browserName is different from the sessionBrowserName |
584-
| relayNode.hpa.platformName | string | `"Android"` | platformName from the capability |
585-
| relayNode.hpa.browserVersion | string | `""` | browserVersion from the capability |
584+
| relayNode.hpa.browserVersion | string | `""` | browserVersion should match with Node stereotype and request capability is scaled by this scaler |
585+
| relayNode.hpa.platformName | string | `"Android"` | platformName should match with Node stereotype and request capability is scaled by this scaler |
586586
| relayNode.hpa.unsafeSsl | string | `"{{ template \"seleniumGrid.graphqlURL.unsafeSsl\" . }}"` | Skip check SSL when connecting to the Graphql endpoint |
587587
| relayNode.initContainers | list | `[]` | It is used to add initContainers in the same pod of the browser node. It should be set using the --set-json option |
588588
| relayNode.sidecars | list | `[]` | It is used to add sidecars proxy in the same pod of the browser node. It means it will add a new container to the deployment itself. It should be set using the --set-json option |

charts/selenium-grid/README.md

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ This chart enables the creation of a Selenium Grid Server in Kubernetes.
1515
* [Settings common for both `job` and `deployment` scalingType](#settings-common-for-both-job-and-deployment-scalingtype)
1616
* [Settings when scalingType with `deployment`](#settings-when-scalingtype-with-deployment-)
1717
* [Settings when scalingType with `job`](#settings-when-scalingtype-with-job)
18+
* [Scaler trigger configuration](#scaler-trigger-configuration)
1819
* [Settings fixed-sized thread pool for the Distributor to create new sessions](#settings-fixed-sized-thread-pool-for-the-distributor-to-create-new-sessions)
1920
* [Updating Selenium-Grid release](#updating-selenium-grid-release)
2021
* [Uninstalling Selenium Grid release](#uninstalling-selenium-grid-release)
@@ -210,6 +211,128 @@ autoscaling:
210211

211212
Settings that KEDA [ScaledJob spec](https://keda.sh/docs/latest/concepts/scaling-jobs/#scaledjob-spec) supports can be set via `autoscaling.scaledJobOptions`.
212213

214+
### Scaler trigger configuration
215+
216+
From KEDA core `v2.16.1+`, the trigger metadata `browserVersion`, `platformName` is recommended to be set explicitly to have the correct scaling behavior (especially when your Grid includes autoscaling Nodes, non-autoscaling Nodes, relay Nodes, etc.). Besides that, in client binding, it is also recommended to set the `browserVersion`, `platformName` to align with the trigger metadata. Please see below examples for more details.
217+
218+
Understand list trigger parameters
219+
220+
- `url` - Graphql url of your Selenium Grid. Refer to the Selenium Grid's documentation [here](https://www.selenium.dev/documentation/en/grid/grid_4/graphql_support/) to for more info. If endpoint requires authentication, you can use `TriggerAuthentication` to provide the credentials instead of embedding in the URL.
221+
- `browserName` - Name of browser that usually gets passed in the browser capability. Refer to the [Selenium Grid's](https://www.selenium.dev/documentation/en/getting_started_with_webdriver/browsers/) and [WebdriverIO's](https://webdriver.io/docs/options/#capabilities) documentation for more info. (Optional)
222+
- `sessionBrowserName` - Name of the browser when it is an active session, only set if `BrowserName` changes between the queue and the active session. See the Edge example below for further detail. (Optional)
223+
- `browserVersion` - Version of browser that usually gets passed in the browser capability. Refer to the [Selenium Grid's](https://www.selenium.dev/documentation/en/getting_started_with_webdriver/browsers/) and [WebdriverIO's](https://webdriver.io/docs/options/#capabilities) documentation for more info. (Optional)
224+
- `platformName` - Name of the browser platform. Refer to the [Selenium Grid's](https://www.selenium.dev/documentation/en/getting_started_with_webdriver/browsers/) and [WebdriverIO's](https://webdriver.io/docs/options/#capabilities) documentation for more info. (Optional)
225+
- `unsafeSsl` - Skip certificate validation when connecting over HTTPS. (Values: `true`, `false`, Default: `false`, Optional)
226+
- `activationThreshold` - Target value for activating the scaler. Learn more about activation [here](./../concepts/scaling-deployments.md#activating-and-scaling-thresholds). (Default: `0`, Optional)
227+
- `nodeMaxSessions` - Number of maximum sessions that can run in parallel on a Node. Update this parameter align with node config `--max-sessions` (`SE_NODE_MAX_SESSIONS`) to have the correct scaling behavior. (Default: `1`, Optional).
228+
229+
Understand list trigger authentication
230+
231+
- `username` - Username for basic authentication in GraphQL endpoint instead of embedding in the URL. (Optional)
232+
- `password` - Password for basic authentication in GraphQL endpoint instead of embedding in the URL. (Optional)
233+
- `authType` - Type of authentication to be used. This can be set to `Bearer` or `OAuth2` in case Selenium Grid behind an Ingress proxy with other authentication types. (Optional)
234+
- `accessToken` - Access token. This is required when `authType` is set a value. (Optional)
235+
236+
In each Node, trigger parameters value will be set under config key `hpa`.
237+
238+
By default, `browserName`, `sessionBrowserName` are set for corresponding node browser. Parameters `browserVersion`, `platformName` are not set, leave them as empty by default. The triggers config looks like
239+
240+
```yaml
241+
triggers:
242+
- type: selenium-grid
243+
metadata:
244+
url: 'http://selenium-hub:4444/graphql'
245+
browserName: 'chrome'
246+
browserVersion: ''
247+
platformName: ''
248+
```
249+
250+
In this case, the scaler will be triggered by below request (example in Python client, common use case that most users get started)
251+
252+
```python
253+
options = ChromeOptions()
254+
driver = webdriver.Remote(options=options, command_executor=SELENIUM_GRID_URL)
255+
```
256+
257+
With above script, the request is sent to Grid. Via GraphQL response, it looks like
258+
259+
```json
260+
{
261+
"data": {
262+
"grid": {
263+
"sessionCount": 0,
264+
"maxSession": 0,
265+
"totalSlots": 0
266+
},
267+
"nodesInfo": {
268+
"nodes": []
269+
},
270+
"sessionsInfo": {
271+
"sessionQueueRequests": [
272+
"{\"browserName\": \"chrome\"}"
273+
]
274+
}
275+
}
276+
}
277+
```
278+
279+
Scaler will trigger to scale up the Node with stereotypes matched to pick up the request in the queue. Via GraphQL response, it looks like
280+
281+
```json
282+
{
283+
"data": {
284+
"grid": {
285+
"sessionCount": 0,
286+
"maxSession": 1,
287+
"totalSlots": 1
288+
},
289+
"nodesInfo": {
290+
"nodes": [
291+
{
292+
"id": "UUID",
293+
"status": "UP",
294+
"sessionCount": 0,
295+
"maxSession": 1,
296+
"slotCount": 1,
297+
"stereotypes": "[{\"slots\": 1, \"stereotype\": {\"browserName\": \"chrome\", \"browserVersion\": \"\", \"platformName\": \"\"}}]",
298+
"sessions": []
299+
}
300+
]
301+
},
302+
"sessionsInfo": {
303+
"sessionQueueRequests": [
304+
"{\"browserName\": \"chrome\"}"
305+
]
306+
}
307+
}
308+
}
309+
```
310+
311+
In Node deployment spec, there is environment variable `SE_NODE_BROWSER_VERSION` which is able to unset `browserVersion` in Node stereotypes (it is setting short browser build number by default e.g `131.0`) or any custom value is up to you, which is expected to match with the request capabilities in queue and scaler trigger metadata.
312+
Similarly, `SE_NODE_PLATFORM_NAME` is used to unset the `platformName` in Node stereotypes if needed. Noted, update to newer image tag if these 2 env variables doesn't take effect for you.
313+
314+
For another example, where your Grid with multiple scalers have different metadata, one of them looks like
315+
316+
```yaml
317+
triggers:
318+
- type: selenium-grid
319+
metadata:
320+
url: 'http://selenium-hub:4444/graphql'
321+
browserName: 'chrome'
322+
browserVersion: '131.0'
323+
platformName: 'Linux'
324+
```
325+
326+
327+
The request to trigger this corresponds to the following Python script
328+
329+
```python
330+
options = ChromeOptions()
331+
options.set_capability('platformName', 'Linux')
332+
options.set_capability('browserVersion', '131.0')
333+
driver = webdriver.Remote(options=options, command_executor=SELENIUM_GRID_URL)
334+
```
335+
213336
### Settings fixed-sized thread pool for the Distributor to create new sessions
214337

215338
When enabling autoscaling, the Distributor might be under a high workload with parallelism tests, which are many requests incoming and nodes scaling up simultaneously. (Refer to: [SeleniumHQ/selenium#13723](https://github.com/SeleniumHQ/selenium/issues/13723)).

0 commit comments

Comments
 (0)