Add WebNN API operations tests #34287
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,246 @@ | ||
| // META: title=test WebNN API clamp operation | ||
| // META: global=window,dedicatedworker | ||
| // META: script=./resources/utils.js | ||
| // META: timeout=long | ||
|
|
||
| 'use strict'; | ||
|
|
||
| // https://webmachinelearning.github.io/webnn/#api-mlgraphbuilder-clamp | ||
|
|
||
| const testClamp = async (operandType, syncFlag, inputValue, inputShape, expected, options) => { | ||
| const x = builder.input('x', {type: operandType, dimensions: inputShape}); | ||
| const y = builder.clamp(x, options); | ||
| const TestTypedArray = TypedArrayDict[operandType]; | ||
| const inputs = {'x': new TestTypedArray(inputValue)}; | ||
| const outputs = {'y': new TestTypedArray(sizeOfShape(inputShape))}; | ||
| let graph; | ||
|
|
||
| if (syncFlag) { | ||
| graph = builder.build({y}); | ||
| context.compute(graph, inputs, outputs); | ||
| } else { | ||
| graph = await builder.buildAsync({y}); | ||
| await context.computeAsync(graph, inputs, outputs); | ||
| } | ||
|
|
||
| assert_array_approx_equals_ulp(outputs.y, expected[operandType], PrecisionMetrics.ULP[operandType].clamp, operandType); | ||
| }; | ||
|
|
||
| // Input data are random float64 numbers between [-10, 10) generated by invoking getRandomArbitrary(-10, 10) | ||
| // refering to https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Math/random#getting_a_random_number_between_two_values | ||
| const inputData = [ | ||
| -3.4356449350874696, -6.530945988411405, -8.175760663838268, 2.0879641317522726, | ||
| -4.480150236948526, -8.591504561715722, 5.071455429211573, -6.618697702258771, | ||
| 4.224577823136105, 6.450272349350044, -8.799923845835664, -3.3445965406946643, | ||
| 5.550524270215341, 1.2788677438688012, 9.333702625514768, 9.2261637863086, | ||
| -7.302720212371034, 1.7865902395032585, 5.564981581526375, 3.145101011211482, | ||
| -8.275078596251655, -1.3557080837143296, 7.348269585030259, -5.530012756488021, | ||
| ]; | ||
| // expected data by clamping input data with default options | ||
| const expectedDefault = { | ||
| float32: [ | ||
|
Contributor
There was a problem hiding this comment. Could you please share how you convert the double precision results of webnn-baseline to float32? |
||
| -3.4356448650360107, -6.530945777893066, -8.175760269165039, 2.0879640579223633, | ||
| -4.48015022277832, -8.591504096984863, 5.071455478668213, -6.618697643280029, | ||
| 4.224577903747559, 6.450272560119629, -8.79992389678955, -3.3445966243743896, | ||
| 5.5505242347717285, 1.2788677215576172, 9.33370304107666, 9.226163864135742, | ||
| -7.302720069885254, 1.7865902185440063, 5.564981460571289, 3.1451010704040527, | ||
| -8.275078773498535, -1.355708122253418, 7.348269462585449, -5.530012607574463, | ||
| ], | ||
| }; | ||
| // expected data by clamping input data within [-5, 7] | ||
| const expectedMinMax = { | ||
| float32: [ | ||
| -3.4356448650360107, -5, -5, 2.0879640579223633, | ||
| -4.48015022277832, -5, 5.071455478668213, -5, | ||
| 4.224577903747559, 6.450272560119629, -5, -3.3445966243743896, | ||
| 5.5505242347717285, 1.2788677215576172, 7, 7, | ||
| -5, 1.7865902185440063, 5.564981460571289, 3.1451010704040527, | ||
| -5, -1.355708122253418, 7, -5, | ||
| ], | ||
| }; | ||
| // expected data by clamping input data within [0, 6] | ||
| const expectedRelu6 = { | ||
| float32: [ | ||
| 0, 0, 0, 2.0879640579223633, | ||
| 0, 0, 5.071455478668213, 0, | ||
| 4.224577903747559, 6, 0, 0, | ||
| 5.5505242347717285, 1.2788677215576172, 6, 6, | ||
| 0, 1.7865902185440063, 5.564981460571289, 3.1451010704040527, | ||
| 0, 0, 6, 0, | ||
| ], | ||
| }; | ||
| // expected data by clamping input data within [1, 8] | ||
| const expectedMinMaxBothPositive = { | ||
| float32: [ | ||
| 1, 1, 1, 2.0879640579223633, | ||
| 1, 1, 5.071455478668213, 1, | ||
| 4.224577903747559, 6.450272560119629, 1, 1, | ||
| 5.5505242347717285, 1.2788677215576172, 8, 8, | ||
| 1, 1.7865902185440063, 5.564981460571289, 3.1451010704040527, | ||
| 1, 1, 7.348269462585449, 1, | ||
| ], | ||
| }; | ||
| // expected data by clamping input data within [-6, -1] | ||
| const expectedMinMaxBothNegative = { | ||
| float32: [ | ||
| -3.4356448650360107, -6, -6, -1, | ||
| -4.48015022277832, -6, -1, -6, | ||
| -1, -1, -6, -3.3445966243743896, | ||
| -1, -1, -1, -1, | ||
| -6, -1, -1, -1, | ||
| -6, -1.355708122253418, -1, -5.530012607574463, | ||
| ], | ||
| }; | ||
|
|
||
| // expected data by clamping input data with specified minValue=-5 | ||
| const expectedMin = { | ||
| float32: [ | ||
| -3.4356448650360107, -5, -5, 2.0879640579223633, | ||
| -4.48015022277832, -5, 5.071455478668213, -5, | ||
| 4.224577903747559, 6.450272560119629, -5, -3.3445966243743896, | ||
| 5.5505242347717285, 1.2788677215576172, 9.33370304107666, 9.226163864135742, | ||
| -5, 1.7865902185440063, 5.564981460571289, 3.1451010704040527, | ||
| -5, -1.355708122253418, 7.348269462585449, -5, | ||
| ], | ||
| }; | ||
| // expected data by clamping input data with specified minValue=0 | ||
| const expectedMinZero = { | ||
| float32: [ | ||
| 0, 0, 0, 2.0879640579223633, | ||
| 0, 0, 5.071455478668213, 0, | ||
| 4.224577903747559, 6.450272560119629, 0, 0, | ||
| 5.5505242347717285, 1.2788677215576172, 9.33370304107666, 9.226163864135742, | ||
| 0, 1.7865902185440063, 5.564981460571289, 3.1451010704040527, | ||
| 0, 0, 7.348269462585449, 0, | ||
| ], | ||
| }; | ||
| // expected data by clamping input data with specified maxValue=7 | ||
| const expectedMax = { | ||
| float32: [ | ||
| -3.4356448650360107, -6.530945777893066, -8.175760269165039, 2.0879640579223633, | ||
| -4.48015022277832, -8.591504096984863, 5.071455478668213, -6.618697643280029, | ||
| 4.224577903747559, 6.450272560119629, -8.79992389678955, -3.3445966243743896, | ||
| 5.5505242347717285, 1.2788677215576172, 7, 7, | ||
| -7.302720069885254, 1.7865902185440063, 5.564981460571289, 3.1451010704040527, | ||
| -8.275078773498535, -1.355708122253418, 7, -5.530012607574463, | ||
| ], | ||
| }; | ||
| // expected data by clamping input data with specified maxValue=0 | ||
| const expectedMaxZero = { | ||
| float32: [ | ||
| -3.4356448650360107, -6.530945777893066, -8.175760269165039, 0, | ||
| -4.48015022277832, -8.591504096984863, 0, -6.618697643280029, | ||
| 0, 0, -8.79992389678955, -3.3445966243743896, | ||
| 0, 0, 0, 0, | ||
| -7.302720069885254, 0, 0, 0, | ||
| -8.275078773498535, -1.355708122253418, 0, -5.530012607574463, | ||
| ], | ||
| }; | ||
|
|
||
| // test = { | ||
| // purpose: [ | ||
| // [inputShape, expectedData, options], | ||
| // ], | ||
| // }; | ||
| const tests = { | ||
| 'default options': [ | ||
| // 1D | ||
| [[24], expectedDefault], | ||
| // 2D | ||
| [[4, 6], expectedDefault], | ||
| // 3D | ||
| [[2, 3, 4], expectedDefault], | ||
| // 4D | ||
| [[2, 3, 2, 2], expectedDefault], | ||
| // 5D | ||
| [[2, 3, 2, 1, 2], expectedDefault], | ||
| ], | ||
| 'only specified minValue option': [ | ||
| // 1D | ||
| [[24], expectedMin, {minValue: -5}], | ||
| [[24], expectedMinZero, {minValue: 0}], | ||
| // 2D | ||
| [[4, 6], expectedMin, {minValue: -5}], | ||
| [[4, 6], expectedMinZero, {minValue: 0}], | ||
| // 3D | ||
| [[2, 3, 4], expectedMin, {minValue: -5}], | ||
| [[2, 3, 4], expectedMinZero, {minValue: 0}], | ||
| // 4D | ||
| [[2, 3, 2, 2], expectedMin, {minValue: -5}], | ||
| [[2, 3, 2, 2], expectedMinZero, {minValue: 0}], | ||
| // 5D | ||
| [[2, 3, 2, 1, 2], expectedMin, {minValue: -5}], | ||
| [[2, 3, 2, 1, 2], expectedMinZero, {minValue: 0}], | ||
| ], | ||
| 'only specified maxValue option': [ | ||
| // 1D | ||
| [[24], expectedMax, {maxValue: 7}], | ||
| [[24], expectedMaxZero, {maxValue: 0}], | ||
| // 2D | ||
| [[4, 6], expectedMax, {maxValue: 7}], | ||
| [[4, 6], expectedMaxZero, {maxValue: 0}], | ||
| // 3D | ||
| [[2, 3, 4], expectedMax, {maxValue: 7}], | ||
| [[2, 3, 4], expectedMaxZero, {maxValue: 0}], | ||
| // 4D | ||
| [[2, 3, 2, 2], expectedMax, {maxValue: 7}], | ||
| [[2, 3, 2, 2], expectedMaxZero, {maxValue: 0}], | ||
| // 5D | ||
| [[2, 3, 2, 1, 2], expectedMax, {maxValue: 7}], | ||
| [[2, 3, 2, 1, 2], expectedMaxZero, {maxValue: 0}], | ||
| ], | ||
| 'both specified minValue and maxValue options': [ | ||
| // 1D | ||
| [[24], expectedMinMax, {minValue: -5, maxValue: 7}], | ||
| [[24], expectedRelu6, {minValue: 0, maxValue: 6}], | ||
| [[24], expectedMinMaxBothPositive, {minValue: 1, maxValue: 8}], | ||
| [[24], expectedMinMaxBothNegative, {minValue: -6, maxValue: -1}], | ||
|
There was a problem hiding this comment. (minor) I see for any expected value, the same options always correspond. e.g. How about putting the options adjacent to their corresponding expected output? e.g. It would be a little easier to update tests then, because you just have to update one place instead of a search and replace. Also the comment becomes mostly moot then "expected data by clamping input data within [0, 6]", since the options are visibly near. |
||
| // 2D | ||
| [[4, 6], expectedMinMax, {minValue: -5, maxValue: 7}], | ||
| [[4, 6], expectedRelu6, {minValue: 0, maxValue: 6}], | ||
| [[4, 6], expectedMinMaxBothPositive, {minValue: 1, maxValue: 8}], | ||
| [[4, 6], expectedMinMaxBothNegative, {minValue: -6, maxValue: -1}], | ||
| // 3D | ||
| [[2, 3, 4], expectedMinMax, {minValue: -5, maxValue: 7}], | ||
| [[2, 3, 4], expectedRelu6, {minValue: 0, maxValue: 6}], | ||
| [[2, 3, 4], expectedMinMaxBothPositive, {minValue: 1, maxValue: 8}], | ||
| [[2, 3, 4], expectedMinMaxBothNegative, {minValue: -6, maxValue: -1}], | ||
| // 4D | ||
| [[2, 3, 2, 2], expectedMinMax, {minValue: -5, maxValue: 7}], | ||
| [[2, 3, 2, 2], expectedRelu6, {minValue: 0, maxValue: 6}], | ||
| [[2, 3, 2, 2], expectedMinMaxBothPositive, {minValue: 1, maxValue: 8}], | ||
| [[2, 3, 2, 2], expectedMinMaxBothNegative, {minValue: -6, maxValue: -1}], | ||
| // 5D | ||
| [[2, 3, 2, 1, 2], expectedMinMax, {minValue: -5, maxValue: 7}], | ||
| [[2, 3, 2, 1, 2], expectedRelu6, {minValue: 0, maxValue: 6}], | ||
| [[2, 3, 2, 1, 2], expectedMinMaxBothPositive, {minValue: 1, maxValue: 8}], | ||
| [[2, 3, 2, 1, 2], expectedMinMaxBothNegative, {minValue: -6, maxValue: -1}], | ||
| ], | ||
|
There was a problem hiding this comment. Should we have at least one case where minValue > 0 and one where maxValue < 0?
Contributor
Author
There was a problem hiding this comment. Do you mean adding exception tests? It's expected that an exception error should be threw for this There was a problem hiding this comment. e.g.
Contributor
Author
There was a problem hiding this comment. I updated clamp tests by adding such tests, please take a look again, thanks. |
||
| }; | ||
| let context; | ||
| let builder; | ||
|
|
||
| ExecutionArray.forEach(executionType => { | ||
| const isSync = executionType === 'sync'; | ||
| if (self.GLOBAL.isWindow() && isSync) { | ||
| return; | ||
| } | ||
|
|
||
| DeviceTypeArray.forEach(deviceType => { | ||
| promise_setup(async () => { | ||
| context = navigator.ml.createContext({deviceType}); | ||
| builder = new MLGraphBuilder(context); | ||
| }); | ||
|
|
||
| OperandTypeArray.forEach(operandType => { | ||
| for (let purpose in tests) { | ||
| promise_test(async () => { | ||
| const subTests = tests[purpose]; | ||
| for (let i = 0; i < subTests.length; i++) { | ||
| await testClamp(operandType, isSync, inputData, subTests[i][0], subTests[i][1], subTests[i][2]); | ||
| } | ||
| }, `test clamp with ${purpose} / ${deviceType} / ${executionType} / ${operandType}`); | ||
| } | ||
| }); | ||
| }); | ||
| }); | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Could you please share how to get those expected data by running https://github.com/webmachinelearning/webnn-baseline against the
inputData? And probably share the script that others can reproduce the expected results.Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
It's also important (even more so) that the input be typed than the output. e.g.
That way both reference engine baseline and browser implementation start from the same value. It's totally fine for the reference engine to immediately upcast the float32 input to float64, then continue all computations in float64, but we just want to avoid any downcoast from float64 to float32 before providing input to the browser implementation.