[Autofill Assistant] Make positioning of highlighted area independent of
viewport height.
This change fixes issues with the event filter of the overlay not
adapting to viewport height changes.
With this change, the position of highlighted elements are reported as
CSS pixels, relative to the page, instead of coordinates relative to the
viewport size. The viewport size and position within the page is tracked
and reported separately.
When displaying or comparing against filter coordinates, CSS pixels are
converted into physical pixels using the viewport width, which allows
adapting to zoom.
This change also simplifies the display of the highlighted area by
ignoring the offset reported to the onScroll methods as it was causing
issues. This allows the model to cache the element position and the
visual viewport position independently; when scrolling, unless the
element position is absolute, element position normally stay the same
and only the viewport changes.
This is the M-76 merge for http://crrev/c/1634737.
(cherry picked from commit 553002120f1f5381f2905f26c17e298b75a452b2)
Bug: 973034
Bug: b/133669408
Bug: b/129050125
Change-Id: I745584b496d960a2d73231238dd2006afb3f7eaf
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1634737
Reviewed-by: Jordan Demeulenaere <jdemeulenaere@chromium.org>
Commit-Queue: Stephane Zermatten <szermatt@chromium.org>
Cr-Original-Commit-Position: refs/heads/master@{#665536}
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1657911
Reviewed-by: Stephane Zermatten <szermatt@chromium.org>
Cr-Commit-Position: refs/branch-heads/3809@{#280}
Cr-Branched-From: d82dec1a818f378c464ba307ddd9c92133eac355-refs/heads/master@{#665002}
diff --git a/chrome/android/features/autofill_assistant/java/src/org/chromium/chrome/browser/autofill_assistant/overlay/AssistantOverlayCoordinator.java b/chrome/android/features/autofill_assistant/java/src/org/chromium/chrome/browser/autofill_assistant/overlay/AssistantOverlayCoordinator.java
index 68b1943..dcee28f 100644
--- a/chrome/android/features/autofill_assistant/java/src/org/chromium/chrome/browser/autofill_assistant/overlay/AssistantOverlayCoordinator.java
+++ b/chrome/android/features/autofill_assistant/java/src/org/chromium/chrome/browser/autofill_assistant/overlay/AssistantOverlayCoordinator.java
@@ -39,6 +39,10 @@
model.addObserver((source, propertyKey) -> {
if (AssistantOverlayModel.STATE == propertyKey) {
setState(model.get(AssistantOverlayModel.STATE));
+ } else if (AssistantOverlayModel.VISUAL_VIEWPORT == propertyKey) {
+ RectF rect = model.get(AssistantOverlayModel.VISUAL_VIEWPORT);
+ mEventFilter.setVisualViewport(rect);
+ mDrawable.setVisualViewport(rect);
} else if (AssistantOverlayModel.TOUCHABLE_AREA == propertyKey) {
List<RectF> area = model.get(AssistantOverlayModel.TOUCHABLE_AREA);
mEventFilter.setTouchableArea(area);
@@ -47,8 +51,6 @@
AssistantOverlayDelegate delegate = model.get(AssistantOverlayModel.DELEGATE);
mEventFilter.setDelegate(delegate);
mDrawable.setDelegate(delegate);
- } else if (AssistantOverlayModel.WEB_CONTENTS == propertyKey) {
- mDrawable.setWebContents(model.get(AssistantOverlayModel.WEB_CONTENTS));
} else if (AssistantOverlayModel.BACKGROUND_COLOR == propertyKey) {
mDrawable.setBackgroundColor(model.get(AssistantOverlayModel.BACKGROUND_COLOR));
} else if (AssistantOverlayModel.HIGHLIGHT_BORDER_COLOR == propertyKey) {
diff --git a/chrome/android/features/autofill_assistant/java/src/org/chromium/chrome/browser/autofill_assistant/overlay/AssistantOverlayDrawable.java b/chrome/android/features/autofill_assistant/java/src/org/chromium/chrome/browser/autofill_assistant/overlay/AssistantOverlayDrawable.java
index de12d78..aadec199 100644
--- a/chrome/android/features/autofill_assistant/java/src/org/chromium/chrome/browser/autofill_assistant/overlay/AssistantOverlayDrawable.java
+++ b/chrome/android/features/autofill_assistant/java/src/org/chromium/chrome/browser/autofill_assistant/overlay/AssistantOverlayDrawable.java
@@ -29,9 +29,6 @@
import org.chromium.chrome.browser.compositor.CompositorViewResizer;
import org.chromium.chrome.browser.fullscreen.ChromeFullscreenManager;
import org.chromium.chrome.browser.fullscreen.ChromeFullscreenManager.FullscreenListener;
-import org.chromium.content_public.browser.GestureListenerManager;
-import org.chromium.content_public.browser.GestureStateListener;
-import org.chromium.content_public.browser.WebContents;
import org.chromium.ui.interpolators.BakedBezierInterpolator;
import java.lang.annotation.Retention;
@@ -49,8 +46,8 @@
* <p>While scrolling, it keeps track of the current scrolling offset and avoids drawing on top of
* the top bar which is can be, during animations, just drawn on top of the compositor.
*/
-class AssistantOverlayDrawable extends Drawable
- implements FullscreenListener, GestureStateListener, CompositorViewResizer.Observer {
+class AssistantOverlayDrawable
+ extends Drawable implements FullscreenListener, CompositorViewResizer.Observer {
private static final int FADE_DURATION_MS = 250;
/** Default background color and alpha. */
@@ -83,7 +80,18 @@
/** When in partial mode, don't draw on {@link #mTransparentArea}. */
private boolean mPartial;
- private List<Box> mTransparentArea = new ArrayList<>();
+ /**
+ * Coordinates of the visual viewport within the page, if known, in CSS pixels relative to the
+ * origin of the page.
+ *
+ * The visual viewport includes the portion of the page that is really visible, excluding any
+ * area not fully visible because of the current zoom value.
+ *
+ * Only relevant in partial mode, when the transparent area is non-empty.
+ */
+ private final RectF mVisualViewport = new RectF();
+
+ private final List<Box> mTransparentArea = new ArrayList<>();
/** Padding added between the element area and the grayed-out area. */
private final float mPaddingPx;
@@ -94,32 +102,6 @@
/** A single RectF instance used for drawing, to avoid creating many instances when drawing. */
private final RectF mDrawRect = new RectF();
- /** True while the browser is scrolling. */
- private boolean mBrowserScrolling;
-
- /**
- * Scrolling offset to use while scrolling right after scrolling.
- *
- * <p>This value shifts the transparent area by that many pixels while scrolling.
- */
- private int mBrowserScrollOffsetY;
-
- /**
- * Offset reported at the beginning of a scroll.
- *
- * <p>This is used to interpret the offsets reported by subsequent calls to {@link
- * #onScrollOffsetOrExtentChanged} or {@link #onScrollEnded}.
- */
- private int mInitialBrowserScrollOffsetY;
-
- /**
- * Current offset that applies on mTransparentArea.
- *
- * <p>This value shifts the transparent area by that many pixels after the end of a scroll and
- * before the next update, which resets this value.
- */
- private int mOffsetY;
-
/**
* Current top margin of this view.
*
@@ -137,7 +119,6 @@
private int mMarginBottom;
private AssistantOverlayDelegate mDelegate;
- private GestureListenerManager mGestureListenerManager;
AssistantOverlayDrawable(Context context, ChromeFullscreenManager fullscreenManager,
CompositorViewResizer viewResizer) {
@@ -203,19 +184,7 @@
mDelegate = delegate;
}
- void setWebContents(@Nullable WebContents webContents) {
- if (mGestureListenerManager != null) {
- mGestureListenerManager.removeListener(this);
- mGestureListenerManager = null;
- }
- if (webContents != null) {
- mGestureListenerManager = GestureListenerManager.fromWebContents(webContents);
- mGestureListenerManager.addListener(this);
- }
- }
-
void destroy() {
- setWebContents(null);
mFullscreenManager.removeListener(this);
mViewResizer.removeObserver(this);
mDelegate = null;
@@ -242,6 +211,11 @@
invalidateSelf();
}
+ void setVisualViewport(RectF visualViewport) {
+ mVisualViewport.set(visualViewport);
+ invalidateSelf();
+ }
+
/** Set or updates the transparent area. */
void setTransparentArea(List<RectF> transparentArea) {
// Add or update boxes for each rectangle in the area.
@@ -276,9 +250,6 @@
}
}
- mOffsetY = 0;
- mInitialBrowserScrollOffsetY += mBrowserScrollOffsetY;
- mBrowserScrollOffsetY = 0;
invalidateSelf();
}
@@ -310,8 +281,14 @@
canvas.drawPaint(mBackground);
+ if (mVisualViewport.isEmpty()) return;
+
+ // Ratio of to use to convert zoomed CSS pixels, to physical pixels. Aspect ratio is
+ // conserved, so width and height are always converted with the same value. Using width
+ // here, since viewport width always corresponds to the overlay width.
+ float cssPixelsToPhysical = ((float) width) / ((float) mVisualViewport.width());
+
int yTop = (int) mFullscreenManager.getContentOffset();
- int height = yBottom - yTop;
for (Box box : mTransparentArea) {
RectF rect = box.getRectToDraw();
if (rect.isEmpty() || (!mPartial && box.mAnimationType != AnimationType.FADE_IN)) {
@@ -322,12 +299,13 @@
int fillAlpha = (int) (mBackgroundAlpha * (1f - box.getVisibility()));
mBoxFill.setAlpha(fillAlpha);
- mDrawRect.left = rect.left * width - mPaddingPx;
+ mDrawRect.left = (rect.left - mVisualViewport.left) * cssPixelsToPhysical - mPaddingPx;
mDrawRect.top =
- yTop + rect.top * height - mPaddingPx - mBrowserScrollOffsetY - mOffsetY;
- mDrawRect.right = rect.right * width + mPaddingPx;
+ yTop + (rect.top - mVisualViewport.top) * cssPixelsToPhysical - mPaddingPx;
+ mDrawRect.right =
+ (rect.right - mVisualViewport.left) * cssPixelsToPhysical + mPaddingPx;
mDrawRect.bottom =
- yTop + rect.bottom * height + mPaddingPx - mBrowserScrollOffsetY - mOffsetY;
+ yTop + (rect.bottom - mVisualViewport.top) * cssPixelsToPhysical + mPaddingPx;
if (mDrawRect.left <= 0 && mDrawRect.right >= width) {
// Rounded corners look strange in the case where the rectangle takes exactly the
// width of the screen.
@@ -362,48 +340,14 @@
@Override
public void onUpdateViewportSize() {
+ askForTouchableAreaUpdate();
invalidateSelf();
}
@Override
public void onHeightChanged(int height) {
- invalidateSelf();
- }
-
- /** Called at the beginning of a scroll gesture triggered by the browser. */
- @Override
- public void onScrollStarted(int scrollOffsetY, int scrollExtentY) {
- mBrowserScrolling = true;
- mInitialBrowserScrollOffsetY = scrollOffsetY;
- mBrowserScrollOffsetY = 0;
- invalidateSelf();
- }
-
- /** Called during a scroll gesture triggered by the browser. */
- @Override
- public void onScrollOffsetOrExtentChanged(int scrollOffsetY, int scrollExtentY) {
- if (!mBrowserScrolling) {
- // onScrollOffsetOrExtentChanged will be called alone, without onScrollStarted during a
- // Javascript-initiated scroll.
- askForTouchableAreaUpdate();
- return;
- }
- mBrowserScrollOffsetY = scrollOffsetY - mInitialBrowserScrollOffsetY;
- invalidateSelf();
askForTouchableAreaUpdate();
- }
-
- /** Called at the end of a scroll gesture triggered by the browser. */
- @Override
- public void onScrollEnded(int scrollOffsetY, int scrollExtentY) {
- if (!mBrowserScrolling) {
- return;
- }
- mOffsetY += (scrollOffsetY - mInitialBrowserScrollOffsetY);
- mBrowserScrollOffsetY = 0;
- mBrowserScrolling = false;
invalidateSelf();
- askForTouchableAreaUpdate();
}
private void askForTouchableAreaUpdate() {
diff --git a/chrome/android/features/autofill_assistant/java/src/org/chromium/chrome/browser/autofill_assistant/overlay/AssistantOverlayEventFilter.java b/chrome/android/features/autofill_assistant/java/src/org/chromium/chrome/browser/autofill_assistant/overlay/AssistantOverlayEventFilter.java
index b05c468..aff95267 100644
--- a/chrome/android/features/autofill_assistant/java/src/org/chromium/chrome/browser/autofill_assistant/overlay/AssistantOverlayEventFilter.java
+++ b/chrome/android/features/autofill_assistant/java/src/org/chromium/chrome/browser/autofill_assistant/overlay/AssistantOverlayEventFilter.java
@@ -59,6 +59,16 @@
*/
private boolean mPartial;
+ /**
+ * Coordinates of the visual viewport within the page, if known, in CSS pixels relative to the
+ * origin of the page. This is used to convert pixel coordinates to CSS coordinates.
+ *
+ * The visual viewport includes the portion of the page that is really visible, excluding any
+ * area not fully visible because of the current zoom value.
+ */
+ private final RectF mVisualViewport = new RectF();
+
+ /** Touchable area, expressed in CSS pixels relative to the layout viewport. */
private List<RectF> mTouchableArea = Collections.emptyList();
/**
@@ -141,6 +151,11 @@
mTouchableArea = touchableArea;
}
+ /** Sets the visual viewport. */
+ void setVisualViewport(RectF visualViewport) {
+ mVisualViewport.set(visualViewport);
+ }
+
@Override
protected boolean onInterceptTouchEventInternal(MotionEvent event, boolean isKeyboardShowing) {
// All events should be sent to onTouchEvent().
@@ -297,8 +312,7 @@
private boolean shouldLetEventThrough(MotionEvent event) {
int yTop = (int) mFullscreenManager.getContentOffset();
int height = mCompositorView.getHeight() - getBottomBarHeight() - yTop;
- return isInTouchableArea(((float) event.getX()) / mCompositorView.getWidth(),
- (((float) event.getY() - yTop) / height));
+ return isInTouchableArea(event.getX(), event.getY() - yTop);
}
/** Considers whether to let the client know about unexpected taps. */
@@ -317,17 +331,18 @@
}
}
- private void askForTouchableAreaUpdate() {
- if (mDelegate != null) {
- mDelegate.updateTouchableArea();
- }
- }
-
private boolean isInTouchableArea(float x, float y) {
+ if (mVisualViewport.isEmpty() || mTouchableArea.isEmpty()) return false;
+
+ // Ratio of to use to convert physical pixels to zoomed CSS pixels. Aspect ratio is
+ // conserved, so width and height are always converted with the same value. Using width
+ // here, since viewport width always corresponds to the overlay width.
+ float physicalPixelsToCss =
+ ((float) mVisualViewport.width()) / ((float) mCompositorView.getWidth());
+ float absoluteXCss = (x * physicalPixelsToCss) + mVisualViewport.left;
+ float absoluteYCss = (y * physicalPixelsToCss) + mVisualViewport.top;
for (RectF rect : mTouchableArea) {
- if (rect.contains(x, y, x, y)) {
- return true;
- }
+ if (rect.contains(absoluteXCss, absoluteYCss)) return true;
}
return false;
}
diff --git a/chrome/android/features/autofill_assistant/java/src/org/chromium/chrome/browser/autofill_assistant/overlay/AssistantOverlayModel.java b/chrome/android/features/autofill_assistant/java/src/org/chromium/chrome/browser/autofill_assistant/overlay/AssistantOverlayModel.java
index 55ed5b1..e212489 100644
--- a/chrome/android/features/autofill_assistant/java/src/org/chromium/chrome/browser/autofill_assistant/overlay/AssistantOverlayModel.java
+++ b/chrome/android/features/autofill_assistant/java/src/org/chromium/chrome/browser/autofill_assistant/overlay/AssistantOverlayModel.java
@@ -10,7 +10,6 @@
import org.chromium.base.annotations.CalledByNative;
import org.chromium.base.annotations.JNINamespace;
-import org.chromium.content_public.browser.WebContents;
import org.chromium.ui.modelutil.PropertyModel;
import java.util.ArrayList;
@@ -23,15 +22,13 @@
public class AssistantOverlayModel extends PropertyModel {
public static final WritableIntPropertyKey STATE = new WritableIntPropertyKey();
- // Skipping equality as a way of fixing offset issues. See b/129048184.
- // TODO(b/129050125): Handle offsets properly and remove.
public static final WritableObjectPropertyKey<List<RectF>> TOUCHABLE_AREA =
- new WritableObjectPropertyKey<>(/* skipEquality= */ true);
-
- public static final WritableObjectPropertyKey<AssistantOverlayDelegate> DELEGATE =
new WritableObjectPropertyKey<>();
- public static final WritableObjectPropertyKey<WebContents> WEB_CONTENTS =
+ public static final WritableObjectPropertyKey<RectF> VISUAL_VIEWPORT =
+ new WritableObjectPropertyKey<>();
+
+ public static final WritableObjectPropertyKey<AssistantOverlayDelegate> DELEGATE =
new WritableObjectPropertyKey<>();
public static final WritableObjectPropertyKey<Integer> BACKGROUND_COLOR =
@@ -41,7 +38,7 @@
new WritableObjectPropertyKey<>();
public AssistantOverlayModel() {
- super(STATE, TOUCHABLE_AREA, DELEGATE, WEB_CONTENTS, BACKGROUND_COLOR,
+ super(STATE, TOUCHABLE_AREA, VISUAL_VIEWPORT, DELEGATE, BACKGROUND_COLOR,
HIGHLIGHT_BORDER_COLOR);
}
@@ -51,6 +48,11 @@
}
@CalledByNative
+ private void setVisualViewport(float left, float top, float right, float bottom) {
+ set(VISUAL_VIEWPORT, new RectF(left, top, right, bottom));
+ }
+
+ @CalledByNative
private void setTouchableArea(float[] coords) {
List<RectF> boxes = new ArrayList<>();
for (int i = 0; i < coords.length; i += 4) {
@@ -66,11 +68,6 @@
}
@CalledByNative
- private void setWebContents(WebContents webContents) {
- set(WEB_CONTENTS, webContents);
- }
-
- @CalledByNative
private boolean setBackgroundColor(String colorString) {
return setColor(BACKGROUND_COLOR, colorString);
}
diff --git a/chrome/browser/android/autofill_assistant/ui_controller_android.cc b/chrome/browser/android/autofill_assistant/ui_controller_android.cc
index e4adf9f..906088e 100644
--- a/chrome/browser/android/autofill_assistant/ui_controller_android.cc
+++ b/chrome/browser/android/autofill_assistant/ui_controller_android.cc
@@ -120,8 +120,6 @@
java_web_contents);
Java_AssistantPaymentRequestModel_setWebContents(
env, GetPaymentRequestModel(), java_web_contents);
- Java_AssistantOverlayModel_setWebContents(env, GetOverlayModel(),
- java_web_contents);
if (ui_delegate->GetState() != AutofillAssistantState::INACTIVE) {
// The UI was created for an existing Controller.
OnStatusMessageChanged(ui_delegate->GetStatusMessage());
@@ -137,7 +135,9 @@
std::vector<RectF> area;
ui_delegate->GetTouchableArea(&area);
- OnTouchableAreaChanged(area);
+ RectF visual_viewport;
+ ui_delegate->GetVisualViewport(&visual_viewport);
+ OnTouchableAreaChanged(visual_viewport, area);
OnResizeViewportChanged(ui_delegate->GetResizeViewport());
OnPeekModeChanged(ui_delegate->GetPeekMode());
OnFormChanged(ui_delegate->GetForm());
@@ -529,8 +529,13 @@
}
void UiControllerAndroid::OnTouchableAreaChanged(
+ const RectF& visual_viewport,
const std::vector<RectF>& areas) {
JNIEnv* env = AttachCurrentThread();
+ Java_AssistantOverlayModel_setVisualViewport(
+ env, GetOverlayModel(), visual_viewport.left, visual_viewport.top,
+ visual_viewport.right, visual_viewport.bottom);
+
std::vector<float> flattened;
for (const auto& rect : areas) {
flattened.emplace_back(rect.left);
diff --git a/chrome/browser/android/autofill_assistant/ui_controller_android.h b/chrome/browser/android/autofill_assistant/ui_controller_android.h
index 9de00b01..cabcf97 100644
--- a/chrome/browser/android/autofill_assistant/ui_controller_android.h
+++ b/chrome/browser/android/autofill_assistant/ui_controller_android.h
@@ -73,7 +73,8 @@
void OnInfoBoxChanged(const InfoBox* info_box) override;
void OnProgressChanged(int progress) override;
void OnProgressVisibilityChanged(bool visible) override;
- void OnTouchableAreaChanged(const std::vector<RectF>& areas) override;
+ void OnTouchableAreaChanged(const RectF& visual_viewport,
+ const std::vector<RectF>& areas) override;
void OnResizeViewportChanged(bool resize_viewport) override;
void OnPeekModeChanged(
ConfigureBottomSheetProto::PeekMode peek_mode) override;
diff --git a/components/autofill_assistant/browser/controller.cc b/components/autofill_assistant/browser/controller.cc
index 04146487..45e6217a7 100644
--- a/components/autofill_assistant/browser/controller.cc
+++ b/components/autofill_assistant/browser/controller.cc
@@ -785,7 +785,7 @@
}
void Controller::UpdateTouchableArea() {
- touchable_element_area()->UpdatePositions();
+ touchable_element_area()->Update();
}
void Controller::OnUserInteractionInsideTouchableArea() {
@@ -947,6 +947,11 @@
touchable_element_area_->GetRectangles(area);
}
+void Controller::GetVisualViewport(RectF* visual_viewport) const {
+ if (touchable_element_area_)
+ touchable_element_area_->GetVisualViewport(visual_viewport);
+}
+
void Controller::OnFatalError(const std::string& error_message,
Metrics::DropOutReason reason) {
LOG(ERROR) << "Autofill Assistant has encountered an error and is shutting "
@@ -1120,8 +1125,9 @@
iter->second == "1";
}
-void Controller::OnTouchableAreaChanged(const std::vector<RectF>& areas) {
- GetUiController()->OnTouchableAreaChanged(areas);
+void Controller::OnTouchableAreaChanged(const RectF& visual_viewport,
+ const std::vector<RectF>& areas) {
+ GetUiController()->OnTouchableAreaChanged(visual_viewport, areas);
}
void Controller::SetPaymentRequestOptions(
diff --git a/components/autofill_assistant/browser/controller.h b/components/autofill_assistant/browser/controller.h
index 7053fada..2455ae14a 100644
--- a/components/autofill_assistant/browser/controller.h
+++ b/components/autofill_assistant/browser/controller.h
@@ -127,6 +127,7 @@
void SetTermsAndConditions(
TermsAndConditionsState terms_and_conditions) override;
void GetTouchableArea(std::vector<RectF>* area) const override;
+ void GetVisualViewport(RectF* visual_viewport) const override;
void OnFatalError(const std::string& error_message,
Metrics::DropOutReason reason) override;
bool GetResizeViewport() override;
@@ -206,7 +207,8 @@
void OnWebContentsFocused(
content::RenderWidgetHost* render_widget_host) override;
- void OnTouchableAreaChanged(const std::vector<RectF>& areas);
+ void OnTouchableAreaChanged(const RectF& visual_viewport,
+ const std::vector<RectF>& areas);
void SelectChip(std::vector<Chip>* chips, int chip_index);
void SetOverlayColors(std::unique_ptr<OverlayColors> colors);
diff --git a/components/autofill_assistant/browser/element_area.cc b/components/autofill_assistant/browser/element_area.cc
index bb55e3a..8a31f6b 100644
--- a/components/autofill_assistant/browser/element_area.cc
+++ b/components/autofill_assistant/browser/element_area.cc
@@ -17,15 +17,14 @@
namespace autofill_assistant {
ElementArea::ElementArea(ScriptExecutorDelegate* delegate)
- : delegate_(delegate), scheduled_update_(false), weak_ptr_factory_(this) {
+ : delegate_(delegate), weak_ptr_factory_(this) {
DCHECK(delegate_);
}
ElementArea::~ElementArea() = default;
void ElementArea::Clear() {
- rectangles_.clear();
- ReportUpdate();
+ SetFromProto(ElementAreaProto());
}
void ElementArea::SetFromProto(const ElementAreaProto& proto) {
@@ -43,32 +42,57 @@
DVLOG(3) << " " << position.selector;
}
}
- ReportUpdate();
- if (rectangles_.empty())
+ if (rectangles_.empty()) {
+ timer_.Stop();
+ ReportUpdate();
return;
+ }
- if (!scheduled_update_) {
- // Check once and schedule regular updates.
- scheduled_update_ = true;
- KeepUpdatingElementPositions();
- } else {
- // If regular updates are already scheduled, just force a check of position
- // right away and keep running the scheduled updates.
- UpdatePositions();
+ Update();
+ if (!timer_.IsRunning()) {
+ timer_.Start(
+ FROM_HERE, delegate_->GetSettings().element_position_update_interval,
+ base::BindRepeating(
+ &ElementArea::Update,
+ // This ElementArea instance owns |update_element_positions_|
+ base::Unretained(this)));
}
}
-void ElementArea::UpdatePositions() {
+void ElementArea::Update() {
if (rectangles_.empty())
return;
+ // If anything is still pending, skip the update.
+ if (visual_viewport_pending_update_)
+ return;
+
+ for (auto& rectangle : rectangles_) {
+ if (rectangle.IsPending())
+ return;
+ }
+
+ // Mark everything as pending at the same time, to avoid reporting partial
+ // results.
+ visual_viewport_pending_update_ = true;
for (auto& rectangle : rectangles_) {
for (auto& position : rectangle.positions) {
// To avoid reporting partial rectangles, all element positions become
// pending at the same time.
position.pending_update = true;
}
+ }
+
+ // Viewport and element positions are always queried, and so reported, at the
+ // same time. This allows supporting both elements whose position is relative
+ // (and move with a scroll) as elements whose position is absolute (and don't
+ // move with a scroll.) Being able to tell the difference would be more
+ // effective and allow refreshing element positions less aggressively.
+ delegate_->GetWebController()->GetVisualViewport(base::BindOnce(
+ &ElementArea::OnGetVisualViewport, weak_ptr_factory_.GetWeakPtr()));
+
+ for (auto& rectangle : rectangles_) {
for (auto& position : rectangle.positions) {
delegate_->GetWebController()->GetElementPosition(
position.selector,
@@ -81,7 +105,7 @@
void ElementArea::GetRectangles(std::vector<RectF>* area) {
for (auto& rectangle : rectangles_) {
area->emplace_back();
- rectangle.FillRect(&area->back());
+ rectangle.FillRect(&area->back(), visual_viewport_);
}
}
@@ -102,11 +126,13 @@
return false;
}
-void ElementArea::Rectangle::FillRect(RectF* rect) const {
+void ElementArea::Rectangle::FillRect(RectF* rect,
+ const RectF& visual_viewport) const {
bool has_first_rect = false;
for (const auto& position : positions) {
- if (position.rect.empty())
+ if (position.rect.empty()) {
continue;
+ }
if (!has_first_rect) {
*rect = position.rect;
@@ -119,26 +145,12 @@
rect->right = std::max(rect->right, position.rect.right);
}
if (has_first_rect && full_width) {
- rect->left = 0.0;
- rect->right = 1.0;
+ rect->left = visual_viewport.left;
+ rect->right = visual_viewport.right;
}
return;
}
-void ElementArea::KeepUpdatingElementPositions() {
- if (rectangles_.empty()) {
- scheduled_update_ = false;
- return;
- }
-
- UpdatePositions();
- base::ThreadTaskRunnerHandle::Get()->PostDelayedTask(
- FROM_HERE,
- base::BindOnce(&ElementArea::KeepUpdatingElementPositions,
- weak_ptr_factory_.GetWeakPtr()),
- delegate_->GetSettings().element_position_update_interval);
-}
-
void ElementArea::OnGetElementPosition(const Selector& selector,
bool found,
const RectF& rect) {
@@ -153,6 +165,7 @@
}
}
}
+
if (updated) {
ReportUpdate();
}
@@ -160,10 +173,35 @@
// rectangles_. This is fine.
}
+void ElementArea::OnGetVisualViewport(bool success, const RectF& rect) {
+ if (!visual_viewport_pending_update_)
+ return;
+
+ visual_viewport_pending_update_ = false;
+ if (!success)
+ return;
+
+ visual_viewport_ = rect;
+ ReportUpdate();
+}
+
void ElementArea::ReportUpdate() {
if (!on_update_)
return;
+ if (rectangles_.empty()) {
+ // Reporting of visual viewport is best effort when reporting empty
+ // rectangles. It might also be empty.
+ on_update_.Run(visual_viewport_, {});
+ return;
+ }
+
+ // If there are rectangles, delay reporting until both the visual viewport
+ // size and the rectangles are available.
+ if (visual_viewport_pending_update_) {
+ return;
+ }
+
for (const auto& rectangle : rectangles_) {
if (rectangle.IsPending()) {
// We don't have everything we need yet
@@ -173,7 +211,8 @@
std::vector<RectF> area;
GetRectangles(&area);
- on_update_.Run(area);
+
+ on_update_.Run(visual_viewport_, area);
}
} // namespace autofill_assistant
diff --git a/components/autofill_assistant/browser/element_area.h b/components/autofill_assistant/browser/element_area.h
index 69c4014..cf13880 100644
--- a/components/autofill_assistant/browser/element_area.h
+++ b/components/autofill_assistant/browser/element_area.h
@@ -11,6 +11,7 @@
#include "base/callback.h"
#include "base/macros.h"
#include "base/memory/weak_ptr.h"
+#include "base/timer/timer.h"
#include "components/autofill_assistant/browser/client_settings.h"
#include "components/autofill_assistant/browser/rectf.h"
#include "components/autofill_assistant/browser/selector.h"
@@ -36,14 +37,14 @@
// The area is updated asynchronously, so Contains will not work right away.
void SetFromProto(const ElementAreaProto& proto);
- // Forces an out-of-schedule update of the positions right away.
+ // Forces an out-of-schedule update of the viewport and positions right away.
//
// This method is never strictly necessary. It is useful to call it when
// there's a reason to think the positions might have changed, to speed up
// updates.
//
// Does nothing if the area is empty.
- void UpdatePositions();
+ void Update();
// Defines a callback that'll be run every time the set of element coordinates
// changes.
@@ -51,7 +52,8 @@
// The argument reports the areas that corresponds to currently known
// elements, which might be empty.
void SetOnUpdate(
- base::RepeatingCallback<void(const std::vector<RectF>& rectangles)> cb) {
+ base::RepeatingCallback<void(const RectF& visual_viewport,
+ const std::vector<RectF>& rectangles)> cb) {
on_update_ = cb;
}
@@ -64,6 +66,12 @@
// Note that the vector is not cleared before rectangles are added.
void GetRectangles(std::vector<RectF>* area);
+ // Gets the coordinates of the visual viewport, in CSS pixels relative to the
+ // layout viewport. Empty if the size of the visual viewport is not known.
+ void GetVisualViewport(RectF* visual_viewport) {
+ *visual_viewport = visual_viewport_;
+ }
+
private:
// A rectangle that corresponds to the area of the visual viewport covered by
// an element. Coordinates are values between 0 and 1, relative to the size of
@@ -97,22 +105,31 @@
bool IsPending() const;
// Fills the given rectangle from the current state, if possible.
- void FillRect(RectF* rect) const;
+ void FillRect(RectF* rect, const RectF& visual_viewport) const;
};
- void KeepUpdatingElementPositions();
void OnGetElementPosition(const Selector& selector,
bool found,
const RectF& rect);
+ void OnGetVisualViewport(bool success, const RectF& rect);
void ReportUpdate();
ScriptExecutorDelegate* const delegate_;
std::vector<Rectangle> rectangles_;
- // If true, regular updates are currently scheduled.
- bool scheduled_update_;
+ // If true, update for the visual viewport position is currently scheduled.
+ bool visual_viewport_pending_update_ = false;
- base::RepeatingCallback<void(const std::vector<RectF>& areas)> on_update_;
+ // Visual viewport coordinates, in CSS pixels, relative to the layout
+ // viewport.
+ RectF visual_viewport_;
+
+ // While running, regularly calls Update().
+ base::RepeatingTimer timer_;
+
+ base::RepeatingCallback<void(const RectF& visual_viewport,
+ const std::vector<RectF>& rectangles)>
+ on_update_;
base::WeakPtrFactory<ElementArea> weak_ptr_factory_;
diff --git a/components/autofill_assistant/browser/element_area_unittest.cc b/components/autofill_assistant/browser/element_area_unittest.cc
index 441ea54..e928994 100644
--- a/components/autofill_assistant/browser/element_area_unittest.cc
+++ b/components/autofill_assistant/browser/element_area_unittest.cc
@@ -71,8 +71,14 @@
base::test::ScopedTaskEnvironment::MainThreadType::MOCK_TIME),
element_area_(&delegate_) {
delegate_.SetWebController(&mock_web_controller_);
+ delegate_.GetMutableSettings()->element_position_update_interval =
+ base::TimeDelta::FromMilliseconds(100);
+
ON_CALL(mock_web_controller_, OnGetElementPosition(_, _))
.WillByDefault(RunOnceCallback<1>(false, RectF()));
+ ON_CALL(mock_web_controller_, OnGetVisualViewport(_))
+ .WillByDefault(RunOnceCallback<0>(true, RectF(0, 0, 200, 400)));
+
element_area_.SetOnUpdate(base::BindRepeating(&ElementAreaTest::OnUpdate,
base::Unretained(this)));
}
@@ -83,7 +89,11 @@
element_area_.SetFromProto(area);
}
- void OnUpdate(const std::vector<RectF>& area) { reported_area_ = area; }
+ void OnUpdate(const RectF& visual_viewport, const std::vector<RectF>& area) {
+ on_update_call_count_++;
+ reported_visual_viewport_ = visual_viewport;
+ reported_area_ = area;
+ }
// scoped_task_environment_ must be first to guarantee other field
// creation run in that environment.
@@ -92,6 +102,8 @@
MockWebController mock_web_controller_;
FakeScriptExecutorDelegate delegate_;
ElementArea element_area_;
+ int on_update_call_count_ = 0;
+ RectF reported_visual_viewport_;
std::vector<RectF> reported_area_;
};
@@ -101,6 +113,10 @@
std::vector<RectF> rectangles;
element_area_.GetRectangles(&rectangles);
EXPECT_THAT(rectangles, IsEmpty());
+
+ RectF viewport;
+ element_area_.GetVisualViewport(&viewport);
+ EXPECT_THAT(viewport, EmptyRectF());
}
TEST_F(ElementAreaTest, ElementNotFound) {
@@ -112,37 +128,72 @@
EXPECT_THAT(rectangles, ElementsAre(EmptyRectF()));
}
+TEST_F(ElementAreaTest, GetVisualViewport) {
+ SetElement("#some_element");
+ RectF viewport;
+ element_area_.GetVisualViewport(&viewport);
+ EXPECT_THAT(viewport, MatchingRectF(0, 0, 200, 400));
+}
+
TEST_F(ElementAreaTest, OneRectangle) {
EXPECT_CALL(mock_web_controller_,
OnGetElementPosition(Eq(Selector({"#found"}).MustBeVisible()), _))
- .WillOnce(RunOnceCallback<1>(true, RectF(0.25f, 0.25f, 0.75f, 0.75f)));
+ .WillOnce(RunOnceCallback<1>(true, RectF(25, 25, 75, 75)));
SetElement("#found");
std::vector<RectF> rectangles;
element_area_.GetRectangles(&rectangles);
- EXPECT_THAT(rectangles,
- ElementsAre(MatchingRectF(0.25f, 0.25f, 0.75f, 0.75f)));
+ EXPECT_THAT(rectangles, ElementsAre(MatchingRectF(25, 25, 75, 75)));
}
TEST_F(ElementAreaTest, CallOnUpdate) {
EXPECT_CALL(mock_web_controller_,
OnGetElementPosition(Eq(Selector({"#found"}).MustBeVisible()), _))
- .WillOnce(RunOnceCallback<1>(true, RectF(0.25f, 0.25f, 0.75f, 0.75f)));
+ .WillOnce(RunOnceCallback<1>(true, RectF(25, 25, 75, 75)));
SetElement("#found");
- EXPECT_THAT(reported_area_,
- ElementsAre(MatchingRectF(0.25f, 0.25f, 0.75f, 0.75f)));
+ EXPECT_EQ(on_update_call_count_, 1);
+ EXPECT_THAT(reported_visual_viewport_, MatchingRectF(0, 0, 200, 400));
+ EXPECT_THAT(reported_area_, ElementsAre(MatchingRectF(25, 25, 75, 75)));
+}
+
+TEST_F(ElementAreaTest, DontCallOnUpdateWhenViewportMissing) {
+ // Swallowing calls to OnGetVisualViewport guarantees that the viewport
+ // position will never be known.
+ EXPECT_CALL(mock_web_controller_, OnGetVisualViewport(_))
+ .WillOnce(DoNothing());
+ EXPECT_CALL(mock_web_controller_,
+ OnGetElementPosition(Eq(Selector({"#found"}).MustBeVisible()), _))
+ .WillOnce(RunOnceCallback<1>(true, RectF(25, 25, 75, 75)));
+
+ SetElement("#found");
+ EXPECT_EQ(on_update_call_count_, 0);
+}
+
+TEST_F(ElementAreaTest, CallOnUpdateWhenViewportMissingAndEmptyRect) {
+ EXPECT_CALL(mock_web_controller_, OnGetVisualViewport(_))
+ .WillRepeatedly(RunOnceCallback<0>(false, RectF()));
+
+ SetElement("#found");
+
+ // A newly empty element area should be reported.
+ on_update_call_count_ = 0;
+ element_area_.Clear();
+
+ EXPECT_EQ(on_update_call_count_, 1);
+ EXPECT_THAT(reported_visual_viewport_, EmptyRectF());
+ EXPECT_THAT(reported_area_, IsEmpty());
}
TEST_F(ElementAreaTest, TwoRectangles) {
EXPECT_CALL(
mock_web_controller_,
OnGetElementPosition(Eq(Selector({"#top_left"}).MustBeVisible()), _))
- .WillOnce(RunOnceCallback<1>(true, RectF(0.0f, 0.0f, 0.25f, 0.25f)));
+ .WillOnce(RunOnceCallback<1>(true, RectF(0, 0, 25, 25)));
EXPECT_CALL(
mock_web_controller_,
OnGetElementPosition(Eq(Selector({"#bottom_right"}).MustBeVisible()), _))
- .WillOnce(RunOnceCallback<1>(true, RectF(0.25f, 0.25f, 1.0f, 1.0f)));
+ .WillOnce(RunOnceCallback<1>(true, RectF(25, 25, 100, 100)));
ElementAreaProto area_proto;
area_proto.add_rectangles()->add_elements()->add_selectors("#top_left");
@@ -151,19 +202,19 @@
std::vector<RectF> rectangles;
element_area_.GetRectangles(&rectangles);
- EXPECT_THAT(rectangles, ElementsAre(MatchingRectF(0.0f, 0.0f, 0.25f, 0.25f),
- MatchingRectF(0.25f, 0.25f, 1.0f, 1.0f)));
+ EXPECT_THAT(rectangles, ElementsAre(MatchingRectF(0, 0, 25, 25),
+ MatchingRectF(25, 25, 100, 100)));
}
TEST_F(ElementAreaTest, OneRectangleTwoElements) {
EXPECT_CALL(
mock_web_controller_,
OnGetElementPosition(Eq(Selector({"#element1"}).MustBeVisible()), _))
- .WillOnce(RunOnceCallback<1>(true, RectF(0.1f, 0.3f, 0.2f, 0.4f)));
+ .WillOnce(RunOnceCallback<1>(true, RectF(1, 3, 2, 4)));
EXPECT_CALL(
mock_web_controller_,
OnGetElementPosition(Eq(Selector({"#element2"}).MustBeVisible()), _))
- .WillOnce(RunOnceCallback<1>(true, RectF(0.5f, 0.2f, 0.6f, 0.5f)));
+ .WillOnce(RunOnceCallback<1>(true, RectF(5, 2, 6, 5)));
ElementAreaProto area_proto;
auto* rectangle_proto = area_proto.add_rectangles();
@@ -173,14 +224,14 @@
std::vector<RectF> rectangles;
element_area_.GetRectangles(&rectangles);
- EXPECT_THAT(rectangles, ElementsAre(MatchingRectF(0.1f, 0.2f, 0.6f, 0.5f)));
+ EXPECT_THAT(rectangles, ElementsAre(MatchingRectF(1, 2, 6, 5)));
}
TEST_F(ElementAreaTest, DoNotReportIncompleteRectangles) {
EXPECT_CALL(
mock_web_controller_,
OnGetElementPosition(Eq(Selector({"#element1"}).MustBeVisible()), _))
- .WillOnce(RunOnceCallback<1>(true, RectF(0.1f, 0.3f, 0.2f, 0.4f)));
+ .WillOnce(RunOnceCallback<1>(true, RectF(1, 3, 2, 4)));
// Getting the position of #element2 neither succeeds nor fails, simulating an
// intermediate state which shouldn't be reported to the callback.
@@ -195,30 +246,30 @@
rectangle_proto->add_elements()->add_selectors("#element2");
element_area_.SetFromProto(area_proto);
- EXPECT_THAT(reported_area_, ElementsAre(EmptyRectF()));
+ EXPECT_THAT(reported_area_, IsEmpty());
std::vector<RectF> rectangles;
element_area_.GetRectangles(&rectangles);
- EXPECT_THAT(rectangles, ElementsAre(MatchingRectF(0.1f, 0.3f, 0.2f, 0.4f)));
+ EXPECT_THAT(rectangles, ElementsAre(MatchingRectF(1, 3, 2, 4)));
}
TEST_F(ElementAreaTest, OneRectangleFourElements) {
EXPECT_CALL(
mock_web_controller_,
OnGetElementPosition(Eq(Selector({"#element1"}).MustBeVisible()), _))
- .WillOnce(RunOnceCallback<1>(true, RectF(0.0f, 0.0f, 0.1f, 0.1f)));
+ .WillOnce(RunOnceCallback<1>(true, RectF(0, 0, 1, 1)));
EXPECT_CALL(
mock_web_controller_,
OnGetElementPosition(Eq(Selector({"#element2"}).MustBeVisible()), _))
- .WillOnce(RunOnceCallback<1>(true, RectF(0.9f, 0.9f, 1.0f, 1.0f)));
+ .WillOnce(RunOnceCallback<1>(true, RectF(9, 9, 100, 100)));
EXPECT_CALL(
mock_web_controller_,
OnGetElementPosition(Eq(Selector({"#element3"}).MustBeVisible()), _))
- .WillOnce(RunOnceCallback<1>(true, RectF(0.0f, 0.9f, 0.1f, 1.0f)));
+ .WillOnce(RunOnceCallback<1>(true, RectF(0, 9, 1, 100)));
EXPECT_CALL(
mock_web_controller_,
OnGetElementPosition(Eq(Selector({"#element4"}).MustBeVisible()), _))
- .WillOnce(RunOnceCallback<1>(true, RectF(0.9f, 0.0f, 1.0f, 0.1f)));
+ .WillOnce(RunOnceCallback<1>(true, RectF(9, 0, 100, 1)));
ElementAreaProto area_proto;
auto* rectangle_proto = area_proto.add_rectangles();
@@ -230,14 +281,14 @@
std::vector<RectF> rectangles;
element_area_.GetRectangles(&rectangles);
- EXPECT_THAT(rectangles, ElementsAre(MatchingRectF(0.0f, 0.0f, 1.0f, 1.0f)));
+ EXPECT_THAT(rectangles, ElementsAre(MatchingRectF(0, 0, 100, 100)));
}
TEST_F(ElementAreaTest, OneRectangleMissingElementsReported) {
EXPECT_CALL(
mock_web_controller_,
OnGetElementPosition(Eq(Selector({"#element1"}).MustBeVisible()), _))
- .WillOnce(RunOnceCallback<1>(true, RectF(0.1f, 0.1f, 0.2f, 0.2f)));
+ .WillOnce(RunOnceCallback<1>(true, RectF(1, 1, 2, 2)));
EXPECT_CALL(
mock_web_controller_,
OnGetElementPosition(Eq(Selector({"#element2"}).MustBeVisible()), _))
@@ -251,21 +302,22 @@
std::vector<RectF> rectangles;
element_area_.GetRectangles(&rectangles);
- EXPECT_THAT(rectangles, ElementsAre(MatchingRectF(0.1f, 0.1f, 0.2f, 0.2f)));
+ EXPECT_THAT(rectangles, ElementsAre(MatchingRectF(1, 1, 2, 2)));
- EXPECT_THAT(reported_area_,
- ElementsAre(MatchingRectF(0.1f, 0.1f, 0.2f, 0.2f)));
+ EXPECT_THAT(reported_area_, ElementsAre(MatchingRectF(1, 1, 2, 2)));
}
TEST_F(ElementAreaTest, FullWidthRectangle) {
EXPECT_CALL(
mock_web_controller_,
OnGetElementPosition(Eq(Selector({"#element1"}).MustBeVisible()), _))
- .WillOnce(RunOnceCallback<1>(true, RectF(0.1f, 0.3f, 0.2f, 0.4f)));
+ .WillOnce(RunOnceCallback<1>(true, RectF(1, 3, 2, 4)));
EXPECT_CALL(
mock_web_controller_,
OnGetElementPosition(Eq(Selector({"#element2"}).MustBeVisible()), _))
- .WillOnce(RunOnceCallback<1>(true, RectF(0.5f, 0.7f, 0.6f, 0.8f)));
+ .WillOnce(RunOnceCallback<1>(true, RectF(5, 7, 6, 8)));
+ EXPECT_CALL(mock_web_controller_, OnGetVisualViewport(_))
+ .WillRepeatedly(RunOnceCallback<0>(true, RectF(100, 0, 200, 400)));
ElementAreaProto area_proto;
auto* rectangle_proto = area_proto.add_rectangles();
@@ -276,7 +328,10 @@
std::vector<RectF> rectangles;
element_area_.GetRectangles(&rectangles);
- EXPECT_THAT(rectangles, ElementsAre(MatchingRectF(0.0f, 0.3f, 1.0f, 0.8f)));
+
+ // left and right of the box come from the visual viewport, top from the 1st
+ // element, bottom from the 2nd.
+ EXPECT_THAT(rectangles, ElementsAre(MatchingRectF(100, 3, 200, 8)));
}
TEST_F(ElementAreaTest, ElementMovesAfterUpdate) {
@@ -284,24 +339,25 @@
EXPECT_CALL(
mock_web_controller_,
OnGetElementPosition(Eq(Selector({"#element"}).MustBeVisible()), _))
- .WillOnce(RunOnceCallback<1>(true, RectF(0.0f, 0.25f, 1.0f, 0.5f)))
- .WillOnce(RunOnceCallback<1>(true, RectF(0.0f, 0.5f, 1.0f, 0.75f)));
+ .WillOnce(RunOnceCallback<1>(true, RectF(0, 25, 100, 50)))
+ .WillOnce(RunOnceCallback<1>(true, RectF(0, 50, 100, 75)));
SetElement("#element");
- EXPECT_THAT(reported_area_,
- ElementsAre(MatchingRectF(0.0f, 0.25f, 1.0f, 0.5f)));
+ std::vector<RectF> original;
+ element_area_.GetRectangles(&original);
+ EXPECT_THAT(original, ElementsAre(MatchingRectF(0, 25, 100, 50)));
+ EXPECT_THAT(reported_area_, ElementsAre(MatchingRectF(0, 25, 100, 50)));
- element_area_.UpdatePositions();
+ element_area_.Update();
// Updated area is available
- std::vector<RectF> rectangles;
- element_area_.GetRectangles(&rectangles);
- EXPECT_THAT(rectangles, ElementsAre(MatchingRectF(0.0f, 0.5f, 1.0f, 0.75f)));
+ std::vector<RectF> updated;
+ element_area_.GetRectangles(&updated);
+ EXPECT_THAT(updated, ElementsAre(MatchingRectF(0, 50, 100, 75)));
// Updated area is reported
- EXPECT_THAT(reported_area_,
- ElementsAre(MatchingRectF(0.0f, 0.5f, 1.0f, 0.75f)));
+ EXPECT_THAT(reported_area_, ElementsAre(MatchingRectF(0, 50, 100, 75)));
}
TEST_F(ElementAreaTest, ElementMovesWithTime) {
@@ -309,13 +365,12 @@
EXPECT_CALL(
mock_web_controller_,
OnGetElementPosition(Eq(Selector({"#element"}).MustBeVisible()), _))
- .WillOnce(RunOnceCallback<1>(true, RectF(0.0f, 0.25f, 1.0f, 0.5f)))
- .WillOnce(RunOnceCallback<1>(true, RectF(0.0f, 0.5f, 1.0f, 0.75f)));
+ .WillOnce(RunOnceCallback<1>(true, RectF(0, 25, 100, 50)))
+ .WillRepeatedly(RunOnceCallback<1>(true, RectF(0, 50, 100, 75)));
SetElement("#element");
- EXPECT_THAT(reported_area_,
- ElementsAre(MatchingRectF(0.0f, 0.25f, 1.0f, 0.5f)));
+ EXPECT_THAT(reported_area_, ElementsAre(MatchingRectF(0, 25, 100, 50)));
scoped_task_environment_.FastForwardBy(
base::TimeDelta::FromMilliseconds(100));
@@ -323,11 +378,10 @@
// Updated area is available
std::vector<RectF> rectangles;
element_area_.GetRectangles(&rectangles);
- EXPECT_THAT(rectangles, ElementsAre(MatchingRectF(0.0f, 0.5f, 1.0f, 0.75f)));
+ EXPECT_THAT(rectangles, ElementsAre(MatchingRectF(0, 50, 100, 75)));
// Updated area is reported
- EXPECT_THAT(reported_area_,
- ElementsAre(MatchingRectF(0.0f, 0.5f, 1.0f, 0.75f)));
+ EXPECT_THAT(reported_area_, ElementsAre(MatchingRectF(0, 50, 100, 75)));
}
} // namespace
} // namespace autofill_assistant
diff --git a/components/autofill_assistant/browser/mock_ui_controller.h b/components/autofill_assistant/browser/mock_ui_controller.h
index a40ea6f..71f9762 100644
--- a/components/autofill_assistant/browser/mock_ui_controller.h
+++ b/components/autofill_assistant/browser/mock_ui_controller.h
@@ -34,7 +34,8 @@
MOCK_METHOD1(OnInfoBoxChanged, void(const InfoBox* info_box));
MOCK_METHOD1(OnProgressChanged, void(int progress));
MOCK_METHOD1(OnProgressVisibilityChanged, void(bool visible));
- MOCK_METHOD1(OnTouchableAreaChanged, void(const std::vector<RectF>& areas));
+ MOCK_METHOD2(OnTouchableAreaChanged,
+ void(const RectF&, const std::vector<RectF>& areas));
MOCK_CONST_METHOD0(Terminate, bool());
MOCK_CONST_METHOD0(GetDropOutReason, Metrics::DropOutReason());
MOCK_METHOD1(OnResizeViewportChanged, void(bool resize_viewport));
diff --git a/components/autofill_assistant/browser/mock_web_controller.h b/components/autofill_assistant/browser/mock_web_controller.h
index 115dcd3..bbfee74a 100644
--- a/components/autofill_assistant/browser/mock_web_controller.h
+++ b/components/autofill_assistant/browser/mock_web_controller.h
@@ -63,6 +63,13 @@
void(const Selector& selector,
base::OnceCallback<void(bool, const std::string&)>& callback));
+ void GetVisualViewport(
+ base::OnceCallback<void(bool, const RectF&)> callback) override {
+ OnGetVisualViewport(callback);
+ }
+ MOCK_METHOD1(OnGetVisualViewport,
+ void(base::OnceCallback<void(bool, const RectF&)>& callback));
+
void GetElementPosition(
const Selector& selector,
base::OnceCallback<void(bool, const RectF&)> callback) override {
diff --git a/components/autofill_assistant/browser/ui_controller.cc b/components/autofill_assistant/browser/ui_controller.cc
index c91e081..a2819e78 100644
--- a/components/autofill_assistant/browser/ui_controller.cc
+++ b/components/autofill_assistant/browser/ui_controller.cc
@@ -25,7 +25,8 @@
void UiController::OnInfoBoxChanged(const InfoBox* info_box) {}
void UiController::OnProgressChanged(int progress) {}
void UiController::OnProgressVisibilityChanged(bool visible) {}
-void UiController::OnTouchableAreaChanged(const std::vector<RectF>& areas) {}
+void UiController::OnTouchableAreaChanged(const RectF& visual_viewport,
+ const std::vector<RectF>& areas) {}
void UiController::OnResizeViewportChanged(bool resize_viewport) {}
void UiController::OnPeekModeChanged(
ConfigureBottomSheetProto::PeekMode peek_mode) {}
diff --git a/components/autofill_assistant/browser/ui_controller.h b/components/autofill_assistant/browser/ui_controller.h
index 2050eef5..adb0342 100644
--- a/components/autofill_assistant/browser/ui_controller.h
+++ b/components/autofill_assistant/browser/ui_controller.h
@@ -76,10 +76,16 @@
// Updates the area of the visible viewport that is accessible when the
// overlay state is OverlayState::PARTIAL.
//
+ // |visual_viewport| contains the position and size of the visual viewport in
+ // the layout viewport. It might be empty if not known or the touchable area
+ // is empty.
+ //
// |rectangles| contains one element per configured rectangles, though these
- // can correspond to empty rectangles. Coordinates are relative to the width
- // or height of the visible viewport, as a number between 0 and 1.
- virtual void OnTouchableAreaChanged(const std::vector<RectF>& rectangles);
+ // can correspond to empty rectangles.
+ //
+ // All rectangles are expressed in absolute CSS coordinates.
+ virtual void OnTouchableAreaChanged(const RectF& visual_viewport,
+ const std::vector<RectF>& rectangles);
// Called when the viewport resize flag has changed.
virtual void OnResizeViewportChanged(bool resize_viewport);
diff --git a/components/autofill_assistant/browser/ui_delegate.h b/components/autofill_assistant/browser/ui_delegate.h
index aa613620..288dd0fc 100644
--- a/components/autofill_assistant/browser/ui_delegate.h
+++ b/components/autofill_assistant/browser/ui_delegate.h
@@ -109,12 +109,16 @@
//
// At the end of this call, |rectangles| contains one element per configured
// rectangles, though these can correspond to empty rectangles. Coordinates
- // are relative to the width or height of the visible viewport, as a number
- // between 0 and 1.
+ // absolute CSS coordinates.
//
// Note that the vector is not cleared before rectangles are added.
virtual void GetTouchableArea(std::vector<RectF>* rectangles) const = 0;
+ // Returns the current size of the visual viewport. May be empty if unknown.
+ //
+ // The rectangle is expressed in absolute CSS coordinates.
+ virtual void GetVisualViewport(RectF* viewport) const = 0;
+
// Reports a fatal error to Autofill Assistant, which should then stop.
virtual void OnFatalError(const std::string& error_message,
Metrics::DropOutReason reason) = 0;
diff --git a/components/autofill_assistant/browser/web_controller.cc b/components/autofill_assistant/browser/web_controller.cc
index 03aca28..458b90b 100644
--- a/components/autofill_assistant/browser/web_controller.cc
+++ b/components/autofill_assistant/browser/web_controller.cc
@@ -46,11 +46,19 @@
const char* const kGetBoundingClientRectAsList =
R"(function(node) {
const r = node.getBoundingClientRect();
- const v = window.visualViewport;
- return [r.left, r.top, r.right, r.bottom,
- v.offsetLeft, v.offsetTop, v.width, v.height];
+ return [window.scrollX + r.left,
+ window.scrollY + r.top,
+ window.scrollX + r.right,
+ window.scrollY + r.bottom];
})";
+const char* const kGetVisualViewport =
+ R"({ const v = window.visualViewport;
+ [v.pageLeft,
+ v.pageTop,
+ v.width,
+ v.height] })";
+
const char* const kScrollIntoViewScript =
R"(function(node) {
node.scrollIntoViewIfNeeded();
@@ -1739,6 +1747,47 @@
weak_ptr_factory_.GetWeakPtr(), std::move(callback)));
}
+void WebController::GetVisualViewport(
+ base::OnceCallback<void(bool, const RectF&)> callback) {
+ devtools_client_->GetRuntime()->Evaluate(
+ runtime::EvaluateParams::Builder()
+ .SetExpression(std::string(kGetVisualViewport))
+ .SetReturnByValue(true)
+ .Build(),
+ base::BindOnce(&WebController::OnGetVisualViewport,
+ weak_ptr_factory_.GetWeakPtr(), std::move(callback)));
+}
+
+void WebController::OnGetVisualViewport(
+ base::OnceCallback<void(bool, const RectF&)> callback,
+ std::unique_ptr<runtime::EvaluateResult> result) {
+ ClientStatus status = CheckJavaScriptResult(result.get(), __FILE__, __LINE__);
+ if (!status.ok() || !result->GetResult()->HasValue() ||
+ !result->GetResult()->GetValue()->is_list() ||
+ result->GetResult()->GetValue()->GetList().size() != 4u) {
+ DVLOG(1) << __func__ << " Failed to get visual viewport: " << status;
+ RectF empty;
+ std::move(callback).Run(false, empty);
+ return;
+ }
+ const auto& list = result->GetResult()->GetValue()->GetList();
+ // Value::GetDouble() is safe to call without checking the value type; it'll
+ // return 0.0 if the value has the wrong type.
+
+ float left = static_cast<float>(list[0].GetDouble());
+ float top = static_cast<float>(list[1].GetDouble());
+ float width = static_cast<float>(list[2].GetDouble());
+ float height = static_cast<float>(list[3].GetDouble());
+
+ RectF rect;
+ rect.left = left;
+ rect.top = top;
+ rect.right = left + width;
+ rect.bottom = top + height;
+
+ std::move(callback).Run(true, rect);
+}
+
void WebController::GetElementPosition(
const Selector& selector,
base::OnceCallback<void(bool, const RectF&)> callback) {
@@ -1776,9 +1825,10 @@
base::OnceCallback<void(bool, const RectF&)> callback,
std::unique_ptr<runtime::CallFunctionOnResult> result) {
ClientStatus status = CheckJavaScriptResult(result.get(), __FILE__, __LINE__);
- if (!status.ok() || !result->GetResult()->GetValue() ||
+ if (!status.ok() || !result->GetResult()->HasValue() ||
!result->GetResult()->GetValue()->is_list() ||
- result->GetResult()->GetValue()->GetList().size() != 8u) {
+ result->GetResult()->GetValue()->GetList().size() != 4u) {
+ DVLOG(2) << __func__ << " Failed to get element position: " << status;
RectF empty;
std::move(callback).Run(false, empty);
return;
@@ -1787,22 +1837,11 @@
// Value::GetDouble() is safe to call without checking the value type; it'll
// return 0.0 if the value has the wrong type.
- // getBoundingClientRect returns coordinates in the layout viewport. They need
- // to be transformed into coordinates in the visual viewport, between 0 and 1.
- float left_layout = static_cast<float>(list[0].GetDouble());
- float top_layout = static_cast<float>(list[1].GetDouble());
- float right_layout = static_cast<float>(list[2].GetDouble());
- float bottom_layout = static_cast<float>(list[3].GetDouble());
- float visual_left_offset = static_cast<float>(list[4].GetDouble());
- float visual_top_offset = static_cast<float>(list[5].GetDouble());
- float visual_w = static_cast<float>(list[6].GetDouble());
- float visual_h = static_cast<float>(list[7].GetDouble());
-
RectF rect;
- rect.left = (left_layout - visual_left_offset) / visual_w;
- rect.top = (top_layout - visual_top_offset) / visual_h;
- rect.right = (right_layout - visual_left_offset) / visual_w;
- rect.bottom = (bottom_layout - visual_top_offset) / visual_h;
+ rect.left = static_cast<float>(list[0].GetDouble());
+ rect.top = static_cast<float>(list[1].GetDouble());
+ rect.right = static_cast<float>(list[2].GetDouble());
+ rect.bottom = static_cast<float>(list[3].GetDouble());
std::move(callback).Run(true, rect);
}
diff --git a/components/autofill_assistant/browser/web_controller.h b/components/autofill_assistant/browser/web_controller.h
index 760fb4d3..2c34de9 100644
--- a/components/autofill_assistant/browser/web_controller.h
+++ b/components/autofill_assistant/browser/web_controller.h
@@ -153,13 +153,18 @@
base::OnceCallback<void(const ClientStatus&, const std::string&)>
callback);
+ // Gets the visual viewport coordinates and size.
+ //
+ // The rectangle is expressed in absolute CSS coordinates.
+ virtual void GetVisualViewport(
+ base::OnceCallback<void(bool, const RectF&)> callback);
+
// Gets the position of the element identified by the selector.
//
// If unsuccessful, the callback gets (false, 0, 0, 0, 0).
//
// If successful, the callback gets (true, left, top, right, bottom), with
- // coordinates expressed as numbers between 0 and 1, relative to the width or
- // height of the visible viewport.
+ // coordinates expressed in absolute CSS coordinates.
virtual void GetElementPosition(
const Selector& selector,
base::OnceCallback<void(bool, const RectF&)> callback);
@@ -400,6 +405,9 @@
base::OnceCallback<void(bool, const RectF&)> callback,
const ClientStatus& status,
std::unique_ptr<FindElementResult> result);
+ void OnGetVisualViewport(
+ base::OnceCallback<void(bool, const RectF&)> callback,
+ std::unique_ptr<runtime::EvaluateResult> result);
void OnGetElementPositionResult(
base::OnceCallback<void(bool, const RectF&)> callback,
std::unique_ptr<runtime::CallFunctionOnResult> result);