fix: refresh display geometry for viewport screenshots

docs/reference/execute-methods.md

@@ -1198,7 +1198,17 @@ The response looks like `{"value":{"left":0,"top":96,"width":828,"height":1696}}
 
 ### mobile: viewportScreenshot
 
-Takes a screenshot of the device viewport (see [`mobile: viewportRect`](#mobile-viewportrect))
+Takes a screenshot of the device viewport (see [`mobile: viewportRect`](#mobile-viewportrect)).
+Native screenshots are cropped using fresh screen information from WDA and the actual image
+bounds, preserving the full width and bottom edge. If WDA reports no top status bar, the
+screenshot is returned unchanged. If the reported status-bar geometry cannot produce a valid
+crop, the original screenshot is returned as a best-effort result. Side-mounted status UI
+cannot be removed by a top-edge crop.
+
+For multiple displays, WDA must provide screen information for the selected display as well
+as capture that display. Wait for folding, rotation, and display-selection transitions to finish
+before requesting a viewport screenshot. Screen information and image capture are separate
+requests, so this command cannot provide an atomic snapshot during a transition.
 
 !!! warning "Unreliable"
 
@@ -1210,7 +1220,7 @@ Base64-encoded string, which represents the viewport screenshot.
 
 ### mobile: deviceScreenInfo
 
-Get information about screen.
+Get current information about the screen from WDA. The driver does not cache this response.
 
 #### Returned Result
 

lib/commands/general.ts

@@ -157,25 +157,25 @@ export async function setUrl(this: XCUITestDriver, url: string): Promise<void> {
  * @returns The viewport rectangle
  */
 export async function getViewportRect(this: XCUITestDriver): Promise<Viewport> {
-  const scale = await this.getDevicePixelRatio();
-  // status bar height comes in unscaled, so scale it
-  const statusBarHeight = Math.trunc((await this.getStatusBarHeight()) * scale);
-  const windowSize = await this.getWindowRect();
+  // Read both values together: display selection, orientation and status bar
+  // visibility can change during a session.
+  const {scale, statusBarSize, screenSize} = await this.getScreenInfo();
+  const statusBarHeight = Math.trunc(statusBarSize.height * scale);
+  const size = screenSize ?? (await this.getWindowRect());
 
   // ios returns coordinates/dimensions in logical pixels, not device pixels,
   // so scale up to device pixels. status bar height is already scaled.
   return {
     left: 0,
     top: statusBarHeight,
-    width: Math.trunc(windowSize.width * scale),
-    height: Math.trunc(windowSize.height * scale) - statusBarHeight,
+    width: Math.trunc(size.width * scale),
+    height: Math.trunc(size.height * scale) - statusBarHeight,
   };
 }
 
 /**
- * Get information about the screen.
+ * Get fresh information about the screen from WDA.
  *
- * @privateRemarks memoized in constructor
  * @returns Screen information including dimensions, scale, and status bar size
  */
 export async function getScreenInfo(this: XCUITestDriver): Promise<ScreenInfo> {
@@ -195,7 +195,6 @@ export async function getStatusBarHeight(this: XCUITestDriver): Promise<number>
 /**
  * Gets the device pixel ratio.
  *
- * @privateRemarks memoized in constructor
  * @returns The device pixel ratio (scale factor)
  */
 export async function getDevicePixelRatio(this: XCUITestDriver): Promise<number> {

lib/commands/screenshots.ts

@@ -110,7 +110,8 @@ export async function getViewportScreenshot(this: XCUITestDriver): Promise<strin
   const screenshot = await this.getScreenshot();
   // if we don't have a status bar, there's nothing to crop, so we can avoid
   // extra calls and return straight away
-  if ((await this.getStatusBarHeight()) === 0) {
+  const {scale, statusBarSize} = await this.getScreenInfo();
+  if (statusBarSize.height === 0) {
     return screenshot;
   }
 
@@ -120,15 +121,18 @@ export async function getViewportScreenshot(this: XCUITestDriver): Promise<strin
     throw new errors.UnableToCaptureScreen('The device screenshot is empty');
   }
   this.log.debug(`Screenshot dimensions: ${width}x${height}`);
-  const region = await this.getViewportRect();
-  if (region.width + region.left > width) {
-    this.log.info('Viewport region exceeds screenshot width, adjusting region to fit');
-    region.width = width - region.left;
-  }
-  if (region.height + region.top > height) {
-    this.log.info('Viewport region exceeds screenshot height, adjusting region to fit');
-    region.height = height - region.top;
+  // A screenshot covers the selected display, which need not have the same
+  // dimensions as the active app window. Preserve its full width and bottom edge,
+  // including fractional-scale rounding in the captured image.
+  const cropHeight = statusBarSize.height * scale;
+  if (!Number.isFinite(cropHeight) || cropHeight < 0 || cropHeight >= height) {
+    this.log.info(
+      `Status bar height ${cropHeight} is outside the screenshot bounds (${width}x${height}); returning the uncropped screenshot`,
+    );
+    return screenshot;
   }
+  const top = Math.trunc(cropHeight);
+  const region = {left: 0, top, width, height: height - top};
   this.log.debug(`Calculated viewport rect: ${JSON.stringify(region)}`);
   return await cropBase64Image(screenshot, region);
 }

lib/commands/types.ts

@@ -428,6 +428,10 @@ export type ContentSizeResult =
   | 'unsupported';
 
 export interface ScreenInfo {
+  /** Display dimensions in logical pixels, when provided by WDA. */
+  screenSize?: Size;
+  /** The display identifier, when provided by WDA. */
+  displayId?: number;
   /**
    * Status bar dimensions
    *

lib/driver.ts

@@ -241,8 +241,6 @@ const NO_PROXY_WEB_LIST: RouteMatcher[] = [
 ] as RouteMatcher[];
 /* eslint-enable no-useless-escape */
 
-const MEMOIZED_FUNCTIONS = ['getStatusBarHeight', 'getDevicePixelRatio', 'getScreenInfo'];
-
 export type XCUITestDriverOpts = DriverOpts<XCUITestDriverConstraints>;
 
 export type W3CXCUITestDriverCaps = W3CDriverCaps<XCUITestDriverConstraints>;
@@ -845,11 +843,6 @@ export class XCUITestDriver
     this._networkMonitorSession = null;
     this._systemMonitorSession = null;
     this._remoteXPCFacade = null;
-    // memoize functions here, so that they are done on a per-instance basis
-    for (const fn of MEMOIZED_FUNCTIONS) {
-      // @ts-expect-error no types
-      this[fn] = memoize(this[fn]);
-    }
     this.lifecycleData = {};
     this._audioRecorder = null;
     this.appInfosCache = new AppInfosCache(this.log);

package.json

@@ -76,7 +76,7 @@
     "appium-ios-device": "^3.1.24",
     "appium-ios-simulator": "^10.7.0",
     "appium-remote-debugger": "^17.4.2",
-    "appium-webdriveragent": "^16.12.11",
+    "appium-webdriveragent": "^16.14.1",
     "appium-xcode": "^7.1.1",
     "async-lock": "^1.4.0",
     "asyncbox": "^6.4.3",

test/unit/commands/general.spec.ts

@@ -1,5 +1,5 @@
 import assert from 'node:assert/strict';
-import {describe, it, beforeEach, afterEach, before} from 'node:test';
+import {describe, it, beforeEach, afterEach} from 'node:test';
 
 import sinon from 'sinon';
 
@@ -161,7 +161,7 @@ describe('general commands', function () {
   });
 
   describe('getDevicePixelRatio and getStatusBarHeight', function () {
-    before(function () {
+    beforeEach(function () {
       mockDriver
         .expects('proxyCommand')
         .withExactArgs('/wda/screen', 'GET')
@@ -182,4 +182,63 @@ describe('general commands', function () {
       assert.strictEqual(await driver.getStatusBarHeight(), 20);
     });
   });
+  describe('screen geometry changes', function () {
+    it('should refresh all geometry accessors when switching displays and returning', async function () {
+      const driver = new XCUITestDriver({} as any);
+      const screen = sinon.stub(driver, 'proxyCommand');
+      for (const info of [
+        {scale: 3, statusBarSize: {width: 466, height: 20}},
+        {scale: 2, statusBarSize: {width: 951, height: 0}},
+        {scale: 3, statusBarSize: {width: 466, height: 24}},
+      ]) {
+        screen.withArgs('/wda/screen', 'GET').resolves(info);
+        assert.deepEqual(await driver.getScreenInfo(), info);
+        assert.equal(await driver.getDevicePixelRatio(), info.scale);
+        assert.equal(await driver.getStatusBarHeight(), info.statusBarSize.height);
+      }
+    });
+
+    it('should retry screen info after a transient error', async function () {
+      const driver = new XCUITestDriver({} as any);
+      const screen = sinon.stub(driver, 'proxyCommand');
+      screen.onFirstCall().rejects(new Error('Display unavailable'));
+      screen.onSecondCall().resolves({scale: 2, statusBarSize: {width: 951, height: 20}});
+      await assert.rejects(driver.getScreenInfo(), /Display unavailable/);
+      assert.equal(await driver.getDevicePixelRatio(), 2);
+    });
+
+    it('should use the display dimensions even when the app window is smaller', async function () {
+      const driver = new XCUITestDriver({} as any);
+      sinon.stub(driver, 'proxyCommand').resolves({
+        scale: 3,
+        statusBarSize: {width: 951, height: 20},
+        screenSize: {width: 951, height: 669},
+      });
+      const window = sinon.stub(driver, 'getWindowRect').resolves({x: 0, y: 0, width: 400, height: 600});
+      assert.deepEqual(await driver.getViewportRect(), {left: 0, top: 60, width: 2853, height: 1947});
+      sinon.assert.notCalled(window);
+    });
+
+    it('should use one screen snapshot per viewport calculation', async function () {
+      const driver = new XCUITestDriver({} as any);
+      const screen = sinon.stub(driver, 'proxyCommand');
+      screen
+        .withArgs('/wda/screen', 'GET')
+        .onFirstCall()
+        .resolves({
+          scale: 2,
+          statusBarSize: {width: 951, height: 20.5},
+        });
+      screen
+        .withArgs('/wda/screen', 'GET')
+        .onSecondCall()
+        .resolves({
+          scale: 3,
+          statusBarSize: {width: 466, height: 30},
+        });
+      sinon.stub(driver, 'getWindowRect').resolves({x: 0, y: 0, width: 951, height: 669});
+      assert.deepEqual(await driver.getViewportRect(), {left: 0, top: 41, width: 1902, height: 1297});
+      sinon.assert.calledOnce(screen);
+    });
+  });
 });

test/unit/commands/screenshots.spec.ts

@@ -2,6 +2,7 @@ import assert from 'node:assert/strict';
 import {describe, it, beforeEach, afterEach} from 'node:test';
 
 import {errors} from 'appium/driver.js';
+import sharp from 'sharp';
 import sinon from 'sinon';
 
 import {XCUITestDriver} from '../../../lib/driver.js';
@@ -95,4 +96,80 @@ describe('screenshots commands', function () {
       });
     });
   });
+  describe('getViewportScreenshot', function () {
+    async function screenshot(width: number, height: number): Promise<string> {
+      return (
+        await sharp({create: {width, height, channels: 3, background: '#abcdef'}})
+          .png()
+          .toBuffer()
+      ).toString('base64');
+    }
+
+    it('should crop each display using fresh geometry and the captured image bounds', async function () {
+      // The image is one pixel larger than the logical window scaled by two.
+      // Cropping must retain the right and bottom edges, including in a smaller app window.
+      const capture = sinon.stub(driver, 'getScreenshot');
+      sinon.stub(driver, 'getWindowRect').resolves({x: 0, y: 0, width: 400, height: 600});
+      for (const [width, height, scale, bar] of [
+        [1398, 2034, 3, 20],
+        [1903, 1339, 2, 24.5],
+        [1903, 1339, 2, 0],
+        [1398, 2034, 3, 24],
+      ]) {
+        const original = await screenshot(width, height);
+        capture.resolves(original);
+        proxyStub.withArgs('/wda/screen', 'GET').resolves({scale, statusBarSize: {width: width / scale, height: bar}});
+        proxyStub.resetHistory();
+        const result = await driver.getViewportScreenshot();
+        const expected = await sharp(Buffer.from(original, 'base64'))
+          .extract({left: 0, top: Math.trunc(bar * scale), width, height: height - Math.trunc(bar * scale)})
+          .raw()
+          .toBuffer();
+        assert.deepEqual(await sharp(Buffer.from(result, 'base64')).raw().toBuffer(), expected);
+        const dimensions = await sharp(Buffer.from(result, 'base64')).metadata();
+        assert.equal(dimensions.width, width);
+        assert.equal(dimensions.height, height - Math.trunc(bar * scale));
+        if (!bar) {
+          assert.equal(result, original);
+        }
+        sinon.assert.calledOnce(proxyStub);
+      }
+    });
+
+    it('should return the original screenshot when status bar geometry cannot be cropped', async function () {
+      // Use a distinct encoding so an accidental full-image crop/re-encode
+      // cannot pass the byte-for-byte preservation assertion.
+      const original = (
+        await sharp(Buffer.from(await screenshot(10, 10), 'base64'))
+          .png({compressionLevel: 0})
+          .toBuffer()
+      ).toString('base64');
+      sinon.stub(driver, 'getScreenshot').resolves(original);
+      for (const [scale, bar] of [
+        [2, 5],
+        [2, 6],
+        [2, -1],
+        [2, -0.1],
+        [0.5, -1],
+        [2, NaN],
+        [Infinity, 1],
+      ]) {
+        proxyStub.resolves({scale, statusBarSize: {width: 5, height: bar}});
+        assert.equal(await driver.getViewportScreenshot(), original);
+      }
+    });
+
+    it('should return web viewport screenshots without requesting native geometry', async function () {
+      sinon.stub(driver, 'isWebContext').returns(true);
+      const capture = sinon.stub(driver, '_webExecutionBackend').get(() => ({
+        screenshot: async () => 'web screenshot',
+      }));
+      try {
+        assert.equal(await driver.getViewportScreenshot(), 'web screenshot');
+        sinon.assert.notCalled(proxyStub);
+      } finally {
+        capture.restore();
+      }
+    });
+  });
 });