Skip to content

Commit

Permalink
[Region Capture #1] Add new produceCropId() API
Browse files Browse the repository at this point in the history
This patch adds a new produceCropId() API to mediaDevices.

This API is called with a DIV or IFRAME element, and adds a new
base::UnguessableToken value to that element's rare data structure.

This token value will be used in followup patches in order to keep track
of an element's location in the page and viewport.

Based on the following design document:
https://docs.google.com/document/d/1dULARMnMZggfWqa_Ti_GrINRNYXGIli3XK9brzAKEV8/

Bug: 1247761
Change-Id: I01cd67e2d4e3dfa7a86289f876e48c8b55095d0a
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3173396
Commit-Queue: Jordan Bayles <jophba@chromium.org>
Reviewed-by: Elad Alon <eladalon@chromium.org>
Reviewed-by: mark a. foltz <mfoltz@chromium.org>
Reviewed-by: Joey Arhar <jarhar@chromium.org>
Cr-Commit-Position: refs/heads/main@{#925544}
  • Loading branch information
baylesj authored and Chromium LUCI CQ committed Sep 28, 2021
1 parent 268ea6b commit 59d29de
Show file tree
Hide file tree
Showing 15 changed files with 231 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -3362,6 +3362,7 @@ enum WebFeature {
kConditionalFocus = 4052,
kV8Navigator_CreateAdRequest_Method = 4053,
kV8Navigator_FinalizeAd_Method = 4054,
kRegionCapture = 4055,

// Add new features immediately above this line. Don't change assigned
// numbers of any item, and don't reuse removed slots.
Expand Down
2 changes: 2 additions & 0 deletions third_party/blink/renderer/bindings/generated_in_core.gni
Original file line number Diff line number Diff line change
Expand Up @@ -1587,6 +1587,8 @@ generated_union_sources_in_core = [
"$root_gen_dir/third_party/blink/renderer/bindings/core/v8/v8_union_file_formdata_usvstring.h",
"$root_gen_dir/third_party/blink/renderer/bindings/core/v8/v8_union_file_usvstring.cc",
"$root_gen_dir/third_party/blink/renderer/bindings/core/v8/v8_union_file_usvstring.h",
"$root_gen_dir/third_party/blink/renderer/bindings/core/v8/v8_union_htmldivelement_htmliframeelement.h",
"$root_gen_dir/third_party/blink/renderer/bindings/core/v8/v8_union_htmldivelement_htmliframeelement.cc",
"$root_gen_dir/third_party/blink/renderer/bindings/core/v8/v8_union_float32array_uint16array_uint8clampedarray.cc",
"$root_gen_dir/third_party/blink/renderer/bindings/core/v8/v8_union_float32array_uint16array_uint8clampedarray.h",
"$root_gen_dir/third_party/blink/renderer/bindings/core/v8/v8_union_float_stringelementrecord.cc",
Expand Down
20 changes: 20 additions & 0 deletions third_party/blink/renderer/core/dom/element.cc
Original file line number Diff line number Diff line change
Expand Up @@ -3626,6 +3626,26 @@ void Element::SetNeedsCompositingUpdate() {
layout_object->Layer()->SetNeedsRepaint();
}

RegionCaptureCropId Element::MarkWithRegionCaptureCropId() {
if (RegionCaptureCropId().is_empty()) {
EnsureElementRareData().SetRegionCaptureCropId(
base::UnguessableToken::Create());

// The crop ID needs to be propagated to the paint system by the time that
// capture begins. The API requires the implementation to propagate the
// token right away, so we force invalidate here.
if (GetLayoutObject()) {
GetLayoutObject()->SetShouldDoFullPaintInvalidation();
}
}
return RegionCaptureCropId();
}

RegionCaptureCropId Element::RegionCaptureCropId() const {
return HasRareData() ? GetElementRareData()->RegionCaptureCropId()
: base::UnguessableToken::Null();
}

void Element::SetCustomElementDefinition(CustomElementDefinition* definition) {
DCHECK(definition);
DCHECK(!GetCustomElementDefinition());
Expand Down
13 changes: 13 additions & 0 deletions third_party/blink/renderer/core/dom/element.h
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@

#include "base/dcheck_is_on.h"
#include "base/gtest_prod_util.h"
#include "base/unguessable_token.h"
#include "third_party/blink/public/common/input/pointer_id.h"
#include "third_party/blink/public/common/metrics/document_update_reason.h"
#include "third_party/blink/renderer/bindings/core/v8/v8_typedefs.h"
Expand Down Expand Up @@ -147,6 +148,8 @@ typedef HeapVector<Member<Attr>> AttrNodeList;

typedef HashMap<AtomicString, SpecificTrustedType> AttrNameToTrustedType;

typedef base::UnguessableToken RegionCaptureCropId;

class CORE_EXPORT Element : public ContainerNode, public Animatable {
DEFINE_WRAPPERTYPEINFO();

Expand Down Expand Up @@ -553,6 +556,16 @@ class CORE_EXPORT Element : public ContainerNode, public Animatable {

void SetNeedsCompositingUpdate();

// Generates a unique crop ID and returns the new value. Not all element
// types have a region capture crop id, however using it here allows access
// to the element rare data struct. Currently, once an element is marked for
// region capture it cannot be unmarked, and repeated calls to this API will
// return the same token.
RegionCaptureCropId MarkWithRegionCaptureCropId();

// Returns a null token if not marked for capture.
RegionCaptureCropId RegionCaptureCropId() const;

ShadowRoot* attachShadow(const ShadowRootInit*, ExceptionState&);

void AttachDeclarativeShadowRoot(HTMLTemplateElement*,
Expand Down
1 change: 1 addition & 0 deletions third_party/blink/renderer/core/dom/element_rare_data.cc
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ struct SameSizeAsElementRareData : NodeRareData {
void* pointers_or_strings[3];
Member<void*> members[18];
bool flags[1];
RegionCaptureCropId crop_id;
};

ElementRareData::ElementRareData(NodeRenderingData* node_layout_data)
Expand Down
11 changes: 11 additions & 0 deletions third_party/blink/renderer/core/dom/element_rare_data.h
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
#define THIRD_PARTY_BLINK_RENDERER_CORE_DOM_ELEMENT_RARE_DATA_H_

#include <memory>
#include "base/unguessable_token.h"
#include "third_party/blink/renderer/core/animation/element_animations.h"
#include "third_party/blink/renderer/core/aom/accessible_node.h"
#include "third_party/blink/renderer/core/css/container_query_data.h"
Expand Down Expand Up @@ -162,6 +163,15 @@ class ElementRareData final : public NodeRareData {
bool HasUndoStack() const { return has_undo_stack_; }
void SetHasUndoStack(bool value) { has_undo_stack_ = value; }

base::UnguessableToken RegionCaptureCropId() const {
return region_capture_token_;
}
void SetRegionCaptureCropId(base::UnguessableToken value) {
DCHECK(!value.is_empty());
DCHECK(region_capture_token_.is_empty());
region_capture_token_ = value;
}

AccessibleNode* GetAccessibleNode() const { return accessible_node_.Get(); }
AccessibleNode* EnsureAccessibleNode(Element* owner_element) {
if (!accessible_node_) {
Expand Down Expand Up @@ -265,6 +275,7 @@ class ElementRareData final : public NodeRareData {
bool should_force_legacy_layout_for_child_ = false;
bool style_should_force_legacy_layout_ = false;
bool has_undo_stack_ = false;
base::UnguessableToken region_capture_token_;
};

inline LayoutSize DefaultMinimumSizeForResizing() {
Expand Down
2 changes: 2 additions & 0 deletions third_party/blink/renderer/core/dom/node.h
Original file line number Diff line number Diff line change
Expand Up @@ -1082,6 +1082,7 @@ class CORE_EXPORT Node : public EventTarget {

bool HasRareData() const { return GetFlag(kHasRareDataFlag); }

// |RareData| cannot be replaced or removed once assigned.
NodeRareData* RareData() const {
SECURITY_DCHECK(HasRareData());
return DataAsNodeRareData();
Expand Down Expand Up @@ -1131,6 +1132,7 @@ class CORE_EXPORT Node : public EventTarget {

void TrackForDebugging();

// Used exclusively by |EnsureRareData|.
NodeRareData& CreateRareData();

const HeapVector<Member<MutationObserverRegistration>>*
Expand Down
29 changes: 29 additions & 0 deletions third_party/blink/renderer/modules/mediastream/media_devices.cc
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,9 @@
#include "third_party/blink/renderer/core/frame/local_frame.h"
#include "third_party/blink/renderer/core/frame/navigator.h"
#include "third_party/blink/renderer/core/frame/web_feature.h"
#include "third_party/blink/renderer/core/html/html_div_element.h"
#include "third_party/blink/renderer/core/html/html_element.h"
#include "third_party/blink/renderer/core/html/html_iframe_element.h"
#include "third_party/blink/renderer/modules/mediastream/identifiability_metrics.h"
#include "third_party/blink/renderer/modules/mediastream/input_device_info.h"
#include "third_party/blink/renderer/modules/mediastream/media_error_state.h"
Expand Down Expand Up @@ -330,6 +333,32 @@ void MediaDevices::setCaptureHandleConfig(ScriptState* script_state,
->SetCaptureHandleConfig(std::move(config_ptr));
}

ScriptPromise MediaDevices::produceCropId(
ScriptState* script_state,
V8UnionHTMLDivElementOrHTMLIFrameElement* element_union,
ExceptionState& exception_state) {
if (!script_state->ContextIsValid()) {
exception_state.ThrowDOMException(DOMExceptionCode::kNotSupportedError,
"Current frame is detached.");
return ScriptPromise();
}

auto* resolver = MakeGarbageCollected<ScriptPromiseResolver>(script_state);
ScriptPromise promise = resolver->Promise();
auto* element =
element_union->IsHTMLDivElement()
? static_cast<Element*>(element_union->GetAsHTMLDivElement())
: static_cast<Element*>(element_union->GetAsHTMLIFrameElement());
const base::UnguessableToken token = element->MarkWithRegionCaptureCropId();
DCHECK(!token.is_empty());

// TODO(crbug.com/1247761): Delay resolution until ack from Viz received.
const std::string stringified_token = token.ToString();
resolver->Resolve(
WTF::String(stringified_token.c_str(), stringified_token.length()));
return promise;
}

const AtomicString& MediaDevices::InterfaceName() const {
return event_target_names::kMediaDevices;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
#include "mojo/public/cpp/bindings/remote.h"
#include "third_party/blink/public/mojom/mediastream/media_devices.mojom-blink.h"
#include "third_party/blink/renderer/bindings/core/v8/active_script_wrappable.h"
#include "third_party/blink/renderer/bindings/core/v8/v8_union_htmldivelement_htmliframeelement.h"
#include "third_party/blink/renderer/core/dom/events/event_target.h"
#include "third_party/blink/renderer/core/execution_context/execution_context_lifecycle_observer.h"
#include "third_party/blink/renderer/modules/event_target_modules.h"
Expand Down Expand Up @@ -67,6 +68,10 @@ class MODULES_EXPORT MediaDevices final
const CaptureHandleConfig*,
ExceptionState&);

ScriptPromise produceCropId(ScriptState*,
V8UnionHTMLDivElementOrHTMLIFrameElement*,
ExceptionState&);

// EventTarget overrides.
const AtomicString& InterfaceName() const override;
ExecutionContext* GetExecutionContext() const override;
Expand Down
11 changes: 11 additions & 0 deletions third_party/blink/renderer/modules/mediastream/media_devices.idl
Original file line number Diff line number Diff line change
Expand Up @@ -36,4 +36,15 @@
RuntimeEnabled = CaptureHandle, CallWith = ScriptState, RaisesException
] void
setCaptureHandleConfig(optional CaptureHandleConfig config = {});

// Given an html element of specified type, produces an identifier for the
// element that can be used by SelfCaptureMediaStreamTrack. Repeated calls
// to this method will return the same value.
// See draft spec here: https://eladalon1983.github.io/region-capture/.
// TODO(crbug.com/1247761): Link to final spec once it's published.
[
CallWith = ScriptState, RaisesException, MeasureAs = RegionCapture,
RuntimeEnabled = RegionCapture
] Promise<DOMString>
produceCropId((HTMLDivElement or HTMLIFrameElement) target);
};
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,13 @@
#include "third_party/blink/public/mojom/media/capture_handle_config.mojom-blink.h"
#include "third_party/blink/renderer/bindings/core/v8/script_function.h"
#include "third_party/blink/renderer/bindings/core/v8/script_promise.h"
#include "third_party/blink/renderer/bindings/core/v8/script_promise_tester.h"
#include "third_party/blink/renderer/bindings/core/v8/v8_binding_for_testing.h"
#include "third_party/blink/renderer/bindings/modules/v8/v8_capture_handle_config.h"
#include "third_party/blink/renderer/bindings/modules/v8/v8_media_stream_constraints.h"
#include "third_party/blink/renderer/core/frame/local_dom_window.h"
#include "third_party/blink/renderer/core/testing/null_execution_context.h"
#include "third_party/blink/renderer/core/testing/page_test_base.h"
#include "third_party/blink/renderer/platform/bindings/exception_state.h"
#include "third_party/blink/renderer/platform/testing/testing_platform_support.h"
#include "third_party/blink/renderer/platform/weborigin/security_origin.h"
Expand Down Expand Up @@ -234,7 +236,7 @@ class MockMediaDevicesDispatcherHost final
mojom::blink::CaptureHandleConfigPtr expected_capture_handle_config_;
};

class MediaDevicesTest : public testing::Test {
class MediaDevicesTest : public PageTestBase {
public:
using MediaDeviceInfos = HeapVector<Member<MediaDeviceInfo>>;

Expand Down Expand Up @@ -704,4 +706,66 @@ TEST_F(MediaDevicesTest,
ToExceptionCode(DOMExceptionCode::kNotSupportedError));
}

TEST_F(MediaDevicesTest, ProduceCropIdWithValidElement) {
V8TestingScope scope;
auto* media_devices = GetMediaDevices(scope.GetWindow());
ASSERT_TRUE(media_devices);

SetBodyContent(R"HTML(
<div id='test-div'></div>
<iframe id='test-iframe' src="about:blank" />
)HTML");

Document& document = GetDocument();
auto div = V8UnionHTMLDivElementOrHTMLIFrameElement(
reinterpret_cast<HTMLDivElement*>(document.getElementById("test-div")));
const ScriptPromise div_promise = media_devices->produceCropId(
scope.GetScriptState(), &div, scope.GetExceptionState());
platform()->RunUntilIdle();
EXPECT_FALSE(div_promise.IsEmpty());
EXPECT_FALSE(scope.GetExceptionState().HadException());

auto iframe = V8UnionHTMLDivElementOrHTMLIFrameElement(
reinterpret_cast<HTMLIFrameElement*>(
document.getElementById("test-iframe")));
const ScriptPromise iframe_promise = media_devices->produceCropId(
scope.GetScriptState(), &iframe, scope.GetExceptionState());
platform()->RunUntilIdle();
EXPECT_FALSE(iframe_promise.IsEmpty());
EXPECT_FALSE(scope.GetExceptionState().HadException());
}

TEST_F(MediaDevicesTest, ProduceCropIdDuplicate) {
V8TestingScope scope;
auto* media_devices = GetMediaDevices(scope.GetWindow());
ASSERT_TRUE(media_devices);

SetBodyContent(R"HTML(
<div id='test-div'></div>
)HTML");

Document& document = GetDocument();
auto div = V8UnionHTMLDivElementOrHTMLIFrameElement(
reinterpret_cast<HTMLDivElement*>(document.getElementById("test-div")));
const ScriptPromise first_promise = media_devices->produceCropId(
scope.GetScriptState(), &div, scope.GetExceptionState());
ScriptPromiseTester first_tester(scope.GetScriptState(), first_promise);
first_tester.WaitUntilSettled();
EXPECT_TRUE(first_tester.IsFulfilled());
EXPECT_FALSE(scope.GetExceptionState().HadException());

// The second call to |produceCropId| should return the same ID.
const ScriptPromise second_promise = media_devices->produceCropId(
scope.GetScriptState(), &div, scope.GetExceptionState());
ScriptPromiseTester second_tester(scope.GetScriptState(), second_promise);
second_tester.WaitUntilSettled();
EXPECT_TRUE(second_tester.IsFulfilled());
EXPECT_FALSE(scope.GetExceptionState().HadException());

WTF::String first_result, second_result;
first_tester.Value().ToString(first_result);
second_tester.Value().ToString(second_result);
EXPECT_EQ(first_result, second_result);
}

} // namespace blink
Original file line number Diff line number Diff line change
Expand Up @@ -1853,6 +1853,10 @@
name: "QuotaChange",
status: "experimental",
},
{
name: "RegionCapture",
status: "experimental",
},
{
name: "RemotePlayback",
status: "stable",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
<!doctype html>
<html>
<head>
<title>Test navigator.mediaDevices.produceCropId()</title>
<meta name='assert' content='Test the produceCropId() method.'/>
</head>

<body>
<h1 class="instructions">Description</h1>
<p class="instructions">This test checks for the behavior of the
<code>navigator.mediaDevices.produceCropId()</code> method.</p>

<div id='test-div'></div>
<iframe id='test-iframe' src="about:blank" />
<a id='test-a'></a>
<div id='log'></div>

<script src=/resources/testharness.js></script>
<script src=/resources/testharnessreport.js></script>

<script>
"use strict";

// Regex that matches a string only if it is exactly 32 valid hex characters.
const HEX_REGEX = /^[0-9A-Fa-f]{32}$/g;
test(() => {
const div_id = document.getElementById('test-div').produceCropId();
assert_true(HEX_REGEX.test(div_id));
}, "produces valid id for div");

test(() => {
const iframe_id = document.getElementById('test-iframe').produceCropId();
assert_true(HEX_REGEX.test(iframe_id));
}, "produces valid id for iframe");

test(() => {
const iframe_id = document.getElementById('test-iframe').produceCropId();
const second_iframe_id = document.getElementById('test-iframe').produceCropId();
assert_equals(iframe_id, second_iframe_id);
}, "repeated calls return the same value");

test(() => {
assert_throws_js(TypeError, function() {
await document.getElementById('test-a').produceCropId();
});
}, "invalid element types cause an error");

test(() => {
const div_id = document.getElementById('test-div').produceCropId();
const iframe_id = document.getElementById('test-iframe').produceCropId();
assert_not_equals(div_id, iframe_id);
}, "two elements have different IDs");

test(() => {
const div = document.getElementById('test-div');
const div_id = div.produceCropId();
const clone = div.cloneNode(true);
document.querySelector('body').appendChild(clone);
const clone_id = clone.produceCropId();
assert_not_equals(div_id, clone_id);
}, "cloned elements have different IDs");

</script>
</body>
</html>
Original file line number Diff line number Diff line change
Expand Up @@ -5254,6 +5254,7 @@ interface MediaDevices : EventTarget
method getDisplayMedia
method getSupportedConstraints
method getUserMedia
method produceCropId
method setCaptureHandleConfig
setter ondevicechange
interface MediaElementAudioSourceNode : AudioNode
Expand Down
1 change: 1 addition & 0 deletions tools/metrics/histograms/enums.xml
Original file line number Diff line number Diff line change
Expand Up @@ -34692,6 +34692,7 @@ Called by update_use_counter_feature_enum.py.-->
<int value="4052" label="ConditionalFocus"/>
<int value="4053" label="V8Navigator_CreateAdRequest_Method"/>
<int value="4054" label="V8Navigator_FinalizeAd_Method"/>
<int value="4055" label="RegionCapture"/>
</enum>

<enum name="FeaturePolicyAllowlistType">
Expand Down

0 comments on commit 59d29de

Please sign in to comment.