feat(cli): add pick and highlight commands

docs/src/api/class-locator.md

@@ -150,6 +150,21 @@ var button = page.GetByRole(AriaRole.Button).And(page.GetByTitle("Subscribe"));
 
 Additional locator to match.
 
+## async method: Locator.ariaRef
+* since: v1.60
+* langs: js
+- returns: <[null]|[string]>
+
+Returns the aria ref (for example `e1`, `e2`) assigned to this element by the most recent aria snapshot, or `null`
+if no ref has been assigned yet. Call [`method: Locator.ariaSnapshot`] or [`method: Page.ariaSnapshot`] before this
+method to ensure a ref is available.
+
+### option: Locator.ariaRef.timeout = %%-input-timeout-%%
+* since: v1.60
+
+### option: Locator.ariaRef.timeout = %%-input-timeout-js-%%
+* since: v1.60
+
 ## async method: Locator.ariaSnapshot
 * since: v1.49
 - returns: <[string]>
@@ -1415,8 +1430,14 @@ Attribute name to get the value for.
 
 ### option: Locator.getByTitle.exact = %%-locator-get-by-text-exact-%%
 
+## async method: Locator.hideHighlight
+* since: v1.60
+
+Hide element highlight added with Highlight the corresponding element(s) on the screen. Useful for debugging, don't commit the code that uses [`method: Locator.highlight`].
+
 ## async method: Locator.highlight
 * since: v1.20
+- returns: <[Disposable]>
 
 Highlight the corresponding element(s) on the screen. Useful for debugging, don't commit the code that uses [`method: Locator.highlight`].
 

packages/injected/src/highlight.ts

@@ -63,6 +63,7 @@ export class Highlight {
   private _injectedScript: InjectedScript;
   private _rafRequest: number | undefined;
   private _language: Language = 'javascript';
+  private _elementHighlightSelectors = new Map<string, ParsedSelector>();
 
   constructor(injectedScript: InjectedScript) {
     this._injectedScript = injectedScript;
@@ -123,22 +124,53 @@ export class Highlight {
     this._language = language;
   }
 
-  runHighlightOnRaf(selector: ParsedSelector) {
+  addElementHighlight(selector: ParsedSelector) {
+    const key = stringifySelector(selector);
+    if (this._elementHighlightSelectors.has(key))
+      return;
+    this._elementHighlightSelectors.set(key, selector);
+    this._ensureElementHighlightRaf();
+  }
+
+  removeElementHighlight(selector: ParsedSelector) {
+    const key = stringifySelector(selector);
+    if (!this._elementHighlightSelectors.delete(key))
+      return;
+    if (this._elementHighlightSelectors.size === 0) {
+      if (this._rafRequest) {
+        this._injectedScript.utils.builtins.cancelAnimationFrame(this._rafRequest);
+        this._rafRequest = undefined;
+      }
+      this.clearHighlight();
+    }
+  }
+
+  private _ensureElementHighlightRaf() {
     if (this._rafRequest)
-      this._injectedScript.utils.builtins.cancelAnimationFrame(this._rafRequest);
-    const elements = this._injectedScript.querySelectorAll(selector, this._injectedScript.document.documentElement);
-    const locator = asLocator(this._language, stringifySelector(selector));
-    const color = elements.length > 1 ? '#f6b26b7f' : '#6fa8dc7f';
-    this.updateHighlight(elements.map((element, index) => {
-      const suffix = elements.length > 1 ? ` [${index + 1} of ${elements.length}]` : '';
-      return { element, color, tooltipText: locator + suffix };
-    }));
-    this._rafRequest = this._injectedScript.utils.builtins.requestAnimationFrame(() => this.runHighlightOnRaf(selector));
+      return;
+    const tick = () => {
+      const entries: HighlightEntry[] = [];
+      for (const selector of this._elementHighlightSelectors.values()) {
+        const elements = this._injectedScript.querySelectorAll(selector, this._injectedScript.document.documentElement);
+        const locator = asLocator(this._language, stringifySelector(selector));
+        const color = elements.length > 1 ? '#f6b26b7f' : '#6fa8dc7f';
+        for (let i = 0; i < elements.length; ++i) {
+          const suffix = elements.length > 1 ? ` [${i + 1} of ${elements.length}]` : '';
+          entries.push({ element: elements[i], color, tooltipText: locator + suffix });
+        }
+      }
+      this.updateHighlight(entries);
+      this._rafRequest = this._injectedScript.utils.builtins.requestAnimationFrame(tick);
+    };
+    this._rafRequest = this._injectedScript.utils.builtins.requestAnimationFrame(tick);
   }
 
   uninstall() {
-    if (this._rafRequest)
+    if (this._rafRequest) {
       this._injectedScript.utils.builtins.cancelAnimationFrame(this._rafRequest);
+      this._rafRequest = undefined;
+    }
+    this._elementHighlightSelectors.clear();
     this._glassPaneElement.remove();
   }
 

packages/injected/src/injectedScript.ts

@@ -1313,12 +1313,14 @@ export class InjectedScript {
     return this._highlight;
   }
 
-  highlight(selector: ParsedSelector) {
-    if (!this._highlight) {
-      this._highlight = new Highlight(this);
-      this._highlight.install();
-    }
-    this._highlight.runHighlightOnRaf(selector);
+  addHighlight(selector: ParsedSelector) {
+    const highlight = this._ensureHighlight();
+    highlight.addElementHighlight(selector);
+  }
+
+  removeHighlight(selector: ParsedSelector) {
+    const highlight = this._ensureHighlight();
+    highlight.removeElementHighlight(selector);
   }
 
   setScreencastAnnotation(annotation: { point?: channels.Point, box?: channels.Rect, actionTitle?: string, duration?: number, position?: string, fontSize?: number } | null) {

packages/isomorphic/protocolMetainfo.ts

@@ -160,6 +160,7 @@ export const methodMetainfo = new Map<string, MethodMetainfo>([
   ['Frame.evalOnSelectorAll', { title: 'Evaluate', snapshot: true, pause: true, }],
   ['Frame.addScriptTag', { title: 'Add script tag', snapshot: true, pause: true, }],
   ['Frame.addStyleTag', { title: 'Add style tag', snapshot: true, pause: true, }],
+  ['Frame.ariaRef', { internal: true, }],
   ['Frame.ariaSnapshot', { title: 'Aria snapshot', group: 'getter', }],
   ['Frame.blur', { title: 'Blur', slowMo: true, snapshot: true, pause: true, }],
   ['Frame.check', { title: 'Check', slowMo: true, snapshot: true, pause: true, input: true, isAutoWaiting: true, }],
@@ -174,7 +175,8 @@ export const methodMetainfo = new Map<string, MethodMetainfo>([
   ['Frame.focus', { title: 'Focus', slowMo: true, snapshot: true, pause: true, }],
   ['Frame.frameElement', { title: 'Get frame element', group: 'getter', }],
   ['Frame.resolveSelector', { internal: true, }],
-  ['Frame.highlight', { title: 'Highlight element', group: 'configuration', }],
+  ['Frame.highlight', { internal: true, }],
+  ['Frame.hideHighlight', { internal: true, }],
   ['Frame.getAttribute', { title: 'Get attribute "{name}"', snapshot: true, pause: true, group: 'getter', }],
   ['Frame.goto', { title: 'Navigate to "{url}"', slowMo: true, snapshot: true, pause: true, }],
   ['Frame.hover', { title: 'Hover', slowMo: true, snapshot: true, pause: true, input: true, isAutoWaiting: true, }],

packages/playwright-client/types/types.d.ts

@@ -12664,6 +12664,24 @@ export interface Locator {
    */
   and(locator: Locator): Locator;
 
+  /**
+   * Returns the aria ref (for example `e1`, `e2`) assigned to this element by the most recent aria snapshot, or `null`
+   * if no ref has been assigned yet. Call
+   * [locator.ariaSnapshot([options])](https://playwright.dev/docs/api/class-locator#locator-aria-snapshot) or
+   * [page.ariaSnapshot([options])](https://playwright.dev/docs/api/class-page#page-aria-snapshot) before this method to
+   * ensure a ref is available.
+   * @param options
+   */
+  ariaRef(options?: {
+    /**
+     * Maximum time in milliseconds. Defaults to `0` - no timeout. The default value can be changed via `actionTimeout`
+     * option in the config, or by using the
+     * [browserContext.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-browsercontext#browser-context-set-default-timeout)
+     * or [page.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-page#page-set-default-timeout) methods.
+     */
+    timeout?: number;
+  }): Promise<null|string>;
+
   /**
    * Captures the aria snapshot of the given element. Read more about [aria snapshots](https://playwright.dev/docs/aria-snapshots) and
    * [expect(locator).toMatchAriaSnapshot(expected[, options])](https://playwright.dev/docs/api/class-locatorassertions#locator-assertions-to-match-aria-snapshot)
@@ -13794,11 +13812,17 @@ export interface Locator {
     exact?: boolean;
   }): Locator;
 
+  /**
+   * Hide element highlight added with Highlight the corresponding element(s) on the screen. Useful for debugging, don't
+   * commit the code that uses [locator.highlight()](https://playwright.dev/docs/api/class-locator#locator-highlight).
+   */
+  hideHighlight(): Promise<void>;
+
   /**
    * Highlight the corresponding element(s) on the screen. Useful for debugging, don't commit the code that uses
    * [locator.highlight()](https://playwright.dev/docs/api/class-locator#locator-highlight).
    */
-  highlight(): Promise<void>;
+  highlight(): Promise<Disposable>;
 
   /**
    * Hover over the matching element.

packages/playwright-core/src/client/frame.ts

@@ -317,6 +317,10 @@ export class Frame extends ChannelOwner<channels.FrameChannel> implements api.Fr
     return await this._channel.highlight({ selector });
   }
 
+  async _hideHighlight(selector: string) {
+    return await this._channel.hideHighlight({ selector });
+  }
+
   locator(selector: string, options?: LocatorOptions): Locator {
     return new Locator(this, selector, options);
   }

packages/playwright-core/src/client/locator.ts

@@ -20,6 +20,7 @@ import { escapeForTextSelector } from '@isomorphic/stringUtils';
 import { isString } from '@isomorphic/rtti';
 import { monotonicTime } from '@isomorphic/time';
 import { ElementHandle } from './elementHandle';
+import { DisposableStub } from './disposable';
 
 import type { Frame } from './frame';
 import type { FilePayload, FrameExpectParams, Rect, SelectOption, SelectOptionOptions, TimeoutOptions } from './types';
@@ -151,7 +152,12 @@ export class Locator implements api.Locator {
   }
 
   async highlight() {
-    return await this._frame._highlight(this._selector);
+    await this._frame._highlight(this._selector);
+    return new DisposableStub(() => this.hideHighlight());
+  }
+
+  async hideHighlight() {
+    await this._frame._hideHighlight(this._selector);
   }
 
   locator(selectorOrLocator: string | Locator, options?: Omit<LocatorOptions, 'visible'>): Locator {
@@ -313,6 +319,11 @@ export class Locator implements api.Locator {
     return await this._withElement((h, timeout) => h.screenshot({ ...options, mask, timeout }), { title: 'Screenshot', timeout: options.timeout });
   }
 
+  async ariaRef(options: TimeoutOptions = {}): Promise<string | null> {
+    const { ref } = await this._frame._channel.ariaRef({ selector: this._selector, timeout: this._frame._timeout(options) });
+    return ref ?? null;
+  }
+
   async ariaSnapshot(options: TimeoutOptions & { mode?: 'ai' | 'default', depth?: number } = {}): Promise<string> {
     const result = await this._frame._channel.ariaSnapshot({ timeout: this._frame._timeout(options), mode: options.mode, selector: this._selector, depth: options.depth });
     return result.snapshot;

packages/playwright-core/src/protocol/validator.ts

@@ -1648,6 +1648,13 @@ scheme.FrameAddStyleTagParams = tObject({
 scheme.FrameAddStyleTagResult = tObject({
   element: tChannel(['ElementHandle']),
 });
+scheme.FrameAriaRefParams = tObject({
+  selector: tString,
+  timeout: tFloat,
+});
+scheme.FrameAriaRefResult = tObject({
+  ref: tOptional(tString),
+});
 scheme.FrameAriaSnapshotParams = tObject({
   mode: tOptional(tEnum(['ai', 'default'])),
   track: tOptional(tString),
@@ -1769,6 +1776,10 @@ scheme.FrameHighlightParams = tObject({
   selector: tString,
 });
 scheme.FrameHighlightResult = tOptional(tObject({}));
+scheme.FrameHideHighlightParams = tObject({
+  selector: tString,
+});
+scheme.FrameHideHighlightResult = tOptional(tObject({}));
 scheme.FrameGetAttributeParams = tObject({
   selector: tString,
   strict: tOptional(tBoolean),

packages/playwright-core/src/server/dispatchers/frameDispatcher.ts

@@ -135,6 +135,10 @@ export class FrameDispatcher extends Dispatcher<Frame, channels.FrameChannel, Br
     return { element: ElementHandleDispatcher.from(this, await this._frame.addStyleTag(progress, params)) };
   }
 
+  async ariaRef(params: channels.FrameAriaRefParams, progress: Progress): Promise<channels.FrameAriaRefResult> {
+    return await this._frame.ariaRef(progress, params.selector);
+  }
+
   async ariaSnapshot(params: channels.FrameAriaSnapshotParams, progress: Progress): Promise<channels.FrameAriaSnapshotResult> {
     return await this._frame.ariaSnapshot(progress, params);
   }
@@ -261,7 +265,11 @@ export class FrameDispatcher extends Dispatcher<Frame, channels.FrameChannel, Br
   }
 
   async highlight(params: channels.FrameHighlightParams, progress: Progress): Promise<void> {
-    return await this._frame.highlight(progress, params.selector);
+    return await this._frame.addHighlight(progress, params.selector);
+  }
+
+  async hideHighlight(params: channels.FrameHideHighlightParams, progress: Progress): Promise<void> {
+    return await this._frame.removeHighlight(progress, params.selector);
   }
 
   async expect(params: channels.FrameExpectParams, progress: Progress): Promise<channels.FrameExpectResult> {

packages/playwright-core/src/server/frames.ts

@@ -1319,12 +1319,21 @@ export class Frame extends SdkObject<FrameEventMap> {
     }, undefined, options, scope);
   }
 
-  async highlight(progress: Progress, selector: string) {
+  async addHighlight(progress: Progress, selector: string) {
     const resolved = await progress.race(this.selectors.resolveInjectedForSelector(selector));
     if (!resolved)
       return;
     return await progress.race(resolved.injected.evaluate((injected, { info }) => {
-      return injected.highlight(info.parsed);
+      return injected.addHighlight(info.parsed);
+    }, { info: resolved.info }));
+  }
+
+  async removeHighlight(progress: Progress, selector: string) {
+    const resolved = await progress.race(this.selectors.resolveInjectedForSelector(selector));
+    if (!resolved)
+      return;
+    return await progress.race(resolved.injected.evaluate((injected, { info }) => {
+      return injected.removeHighlight(info.parsed);
     }, { info: resolved.info }));
   }
 
@@ -1746,6 +1755,15 @@ export class Frame extends SdkObject<FrameEventMap> {
     }, { source, arg });
   }
 
+  async ariaRef(progress: Progress, selector: string): Promise<{ ref?: string }> {
+    const ref = await this._retryWithProgressIfNotConnected(progress, selector, { strict: true, performActionPreChecks: true }, async (progress, handle) => {
+      return await progress.race(handle.evaluateInUtility(([injected, element]) => {
+        return (element as any)._ariaRef?.ref as string | undefined;
+      }, {}));
+    });
+    return { ref };
+  }
+
   async ariaSnapshot(progress: Progress, options: { mode?: 'ai' | 'default', track?: string, doNotRenderActive?: boolean, selector?: string, depth?: number } = {}): Promise<{ snapshot: string }> {
     if (options.selector && options.track)
       throw new Error('Cannot specify both selector and track options');

packages/playwright-core/src/tools/backend/devtools.ts

@@ -15,7 +15,8 @@
  */
 
 import * as z from 'zod';
-import { defineTool } from './tool';
+import { defineTabTool, defineTool } from './tool';
+import { elementSchema } from './snapshot';
 
 const resume = defineTool({
   capability: 'devtools',
@@ -64,4 +65,58 @@ const resume = defineTool({
   },
 });
 
-export default [resume];
+const pickLocator = defineTabTool({
+  capability: 'devtools',
+  schema: {
+    name: 'browser_pick_locator',
+    title: 'Pick element locator',
+    description: 'Wait for the user to pick an element in the browser and return its ref and locator.',
+    inputSchema: z.object({}),
+    type: 'readOnly',
+  },
+
+  handle: async (tab, params, response) => {
+    const locator = await tab.page.pickLocator();
+    // Regenerate aria refs so the picked element has an aria ref we can read below.
+    await tab.page.ariaSnapshot({ mode: 'ai' });
+    const ref = await locator.ariaRef();
+    const resolved = await locator.normalize();
+    response.addTextResult(`ref: ${ref ?? '(none)'}\nlocator: ${resolved.toString()}`);
+  },
+});
+
+const highlight = defineTabTool({
+  capability: 'devtools',
+  schema: {
+    name: 'browser_highlight',
+    title: 'Highlight element',
+    description: 'Show a persistent highlight overlay around the element on the page.',
+    inputSchema: elementSchema,
+    type: 'readOnly',
+  },
+
+  handle: async (tab, params, response) => {
+    const { locator, resolved } = await tab.refLocator(params);
+    await (await locator.normalize()).highlight();
+    response.addTextResult(`Highlighted ${resolved}`);
+  },
+});
+
+const hideHighlight = defineTabTool({
+  capability: 'devtools',
+  schema: {
+    name: 'browser_hide_highlight',
+    title: 'Hide element highlight',
+    description: 'Remove a highlight overlay previously added for the element.',
+    inputSchema: elementSchema,
+    type: 'readOnly',
+  },
+
+  handle: async (tab, params, response) => {
+    const { locator, resolved } = await tab.refLocator(params);
+    await (await locator.normalize()).hideHighlight();
+    response.addTextResult(`Hid highlight for ${resolved}`);
+  },
+});
+
+export default [resume, pickLocator, highlight, hideHighlight];