diff --git a/docs/reference/execute-methods.md b/docs/reference/execute-methods.md index 10da21bea1..2443805c50 100644 --- a/docs/reference/execute-methods.md +++ b/docs/reference/execute-methods.md @@ -1198,7 +1198,17 @@ The response looks like `{"value":{"left":0,"top":96,"width":828,"height":1696}} ### mobile: viewportScreenshot -Takes a screenshot of the device viewport (see [`mobile: viewportRect`](#mobile-viewportrect)) +Takes a screenshot of the device viewport (see [`mobile: viewportRect`](#mobile-viewportrect)). +Native screenshots are cropped using fresh screen information from WDA and the actual image +bounds, preserving the full width and bottom edge. If WDA reports no top status bar, the +screenshot is returned unchanged. If the reported status-bar geometry cannot produce a valid +crop, the original screenshot is returned as a best-effort result. Side-mounted status UI +cannot be removed by a top-edge crop. + +For multiple displays, WDA must provide screen information for the selected display as well +as capture that display. Wait for folding, rotation, and display-selection transitions to finish +before requesting a viewport screenshot. Screen information and image capture are separate +requests, so this command cannot provide an atomic snapshot during a transition. !!! warning "Unreliable" @@ -1210,7 +1220,7 @@ Base64-encoded string, which represents the viewport screenshot. ### mobile: deviceScreenInfo -Get information about screen. +Get current information about the screen from WDA. The driver does not cache this response. #### Returned Result diff --git a/lib/commands/general.ts b/lib/commands/general.ts index be1624d9c4..be770de4f4 100644 --- a/lib/commands/general.ts +++ b/lib/commands/general.ts @@ -157,25 +157,25 @@ export async function setUrl(this: XCUITestDriver, url: string): Promise { * @returns The viewport rectangle */ export async function getViewportRect(this: XCUITestDriver): Promise { - const scale = await this.getDevicePixelRatio(); - // status bar height comes in unscaled, so scale it - const statusBarHeight = Math.trunc((await this.getStatusBarHeight()) * scale); - const windowSize = await this.getWindowRect(); + // Read both values together: display selection, orientation and status bar + // visibility can change during a session. + const {scale, statusBarSize, screenSize} = await this.getScreenInfo(); + const statusBarHeight = Math.trunc(statusBarSize.height * scale); + const size = screenSize ?? (await this.getWindowRect()); // ios returns coordinates/dimensions in logical pixels, not device pixels, // so scale up to device pixels. status bar height is already scaled. return { left: 0, top: statusBarHeight, - width: Math.trunc(windowSize.width * scale), - height: Math.trunc(windowSize.height * scale) - statusBarHeight, + width: Math.trunc(size.width * scale), + height: Math.trunc(size.height * scale) - statusBarHeight, }; } /** - * Get information about the screen. + * Get fresh information about the screen from WDA. * - * @privateRemarks memoized in constructor * @returns Screen information including dimensions, scale, and status bar size */ export async function getScreenInfo(this: XCUITestDriver): Promise { @@ -195,7 +195,6 @@ export async function getStatusBarHeight(this: XCUITestDriver): Promise /** * Gets the device pixel ratio. * - * @privateRemarks memoized in constructor * @returns The device pixel ratio (scale factor) */ export async function getDevicePixelRatio(this: XCUITestDriver): Promise { diff --git a/lib/commands/screenshots.ts b/lib/commands/screenshots.ts index dea0ef0394..a13f065348 100644 --- a/lib/commands/screenshots.ts +++ b/lib/commands/screenshots.ts @@ -110,7 +110,8 @@ export async function getViewportScreenshot(this: XCUITestDriver): Promise width) { - this.log.info('Viewport region exceeds screenshot width, adjusting region to fit'); - region.width = width - region.left; - } - if (region.height + region.top > height) { - this.log.info('Viewport region exceeds screenshot height, adjusting region to fit'); - region.height = height - region.top; + // A screenshot covers the selected display, which need not have the same + // dimensions as the active app window. Preserve its full width and bottom edge, + // including fractional-scale rounding in the captured image. + const cropHeight = statusBarSize.height * scale; + if (!Number.isFinite(cropHeight) || cropHeight < 0 || cropHeight >= height) { + this.log.info( + `Status bar height ${cropHeight} is outside the screenshot bounds (${width}x${height}); returning the uncropped screenshot`, + ); + return screenshot; } + const top = Math.trunc(cropHeight); + const region = {left: 0, top, width, height: height - top}; this.log.debug(`Calculated viewport rect: ${JSON.stringify(region)}`); return await cropBase64Image(screenshot, region); } diff --git a/lib/commands/types.ts b/lib/commands/types.ts index 5fd9b01c54..66666fca23 100644 --- a/lib/commands/types.ts +++ b/lib/commands/types.ts @@ -428,6 +428,10 @@ export type ContentSizeResult = | 'unsupported'; export interface ScreenInfo { + /** Display dimensions in logical pixels, when provided by WDA. */ + screenSize?: Size; + /** The display identifier, when provided by WDA. */ + displayId?: number; /** * Status bar dimensions * diff --git a/lib/driver.ts b/lib/driver.ts index 622d08939f..0a3206b382 100644 --- a/lib/driver.ts +++ b/lib/driver.ts @@ -241,8 +241,6 @@ const NO_PROXY_WEB_LIST: RouteMatcher[] = [ ] as RouteMatcher[]; /* eslint-enable no-useless-escape */ -const MEMOIZED_FUNCTIONS = ['getStatusBarHeight', 'getDevicePixelRatio', 'getScreenInfo']; - export type XCUITestDriverOpts = DriverOpts; export type W3CXCUITestDriverCaps = W3CDriverCaps; @@ -845,11 +843,6 @@ export class XCUITestDriver this._networkMonitorSession = null; this._systemMonitorSession = null; this._remoteXPCFacade = null; - // memoize functions here, so that they are done on a per-instance basis - for (const fn of MEMOIZED_FUNCTIONS) { - // @ts-expect-error no types - this[fn] = memoize(this[fn]); - } this.lifecycleData = {}; this._audioRecorder = null; this.appInfosCache = new AppInfosCache(this.log); diff --git a/package.json b/package.json index 8e3e5853de..cef912bc9f 100644 --- a/package.json +++ b/package.json @@ -76,7 +76,7 @@ "appium-ios-device": "^3.1.24", "appium-ios-simulator": "^10.7.0", "appium-remote-debugger": "^17.4.2", - "appium-webdriveragent": "^16.12.11", + "appium-webdriveragent": "^16.14.1", "appium-xcode": "^7.1.1", "async-lock": "^1.4.0", "asyncbox": "^6.4.3", diff --git a/test/unit/commands/general.spec.ts b/test/unit/commands/general.spec.ts index 8dca7c8be1..adf81d84ff 100644 --- a/test/unit/commands/general.spec.ts +++ b/test/unit/commands/general.spec.ts @@ -1,5 +1,5 @@ import assert from 'node:assert/strict'; -import {describe, it, beforeEach, afterEach, before} from 'node:test'; +import {describe, it, beforeEach, afterEach} from 'node:test'; import sinon from 'sinon'; @@ -161,7 +161,7 @@ describe('general commands', function () { }); describe('getDevicePixelRatio and getStatusBarHeight', function () { - before(function () { + beforeEach(function () { mockDriver .expects('proxyCommand') .withExactArgs('/wda/screen', 'GET') @@ -182,4 +182,63 @@ describe('general commands', function () { assert.strictEqual(await driver.getStatusBarHeight(), 20); }); }); + describe('screen geometry changes', function () { + it('should refresh all geometry accessors when switching displays and returning', async function () { + const driver = new XCUITestDriver({} as any); + const screen = sinon.stub(driver, 'proxyCommand'); + for (const info of [ + {scale: 3, statusBarSize: {width: 466, height: 20}}, + {scale: 2, statusBarSize: {width: 951, height: 0}}, + {scale: 3, statusBarSize: {width: 466, height: 24}}, + ]) { + screen.withArgs('/wda/screen', 'GET').resolves(info); + assert.deepEqual(await driver.getScreenInfo(), info); + assert.equal(await driver.getDevicePixelRatio(), info.scale); + assert.equal(await driver.getStatusBarHeight(), info.statusBarSize.height); + } + }); + + it('should retry screen info after a transient error', async function () { + const driver = new XCUITestDriver({} as any); + const screen = sinon.stub(driver, 'proxyCommand'); + screen.onFirstCall().rejects(new Error('Display unavailable')); + screen.onSecondCall().resolves({scale: 2, statusBarSize: {width: 951, height: 20}}); + await assert.rejects(driver.getScreenInfo(), /Display unavailable/); + assert.equal(await driver.getDevicePixelRatio(), 2); + }); + + it('should use the display dimensions even when the app window is smaller', async function () { + const driver = new XCUITestDriver({} as any); + sinon.stub(driver, 'proxyCommand').resolves({ + scale: 3, + statusBarSize: {width: 951, height: 20}, + screenSize: {width: 951, height: 669}, + }); + const window = sinon.stub(driver, 'getWindowRect').resolves({x: 0, y: 0, width: 400, height: 600}); + assert.deepEqual(await driver.getViewportRect(), {left: 0, top: 60, width: 2853, height: 1947}); + sinon.assert.notCalled(window); + }); + + it('should use one screen snapshot per viewport calculation', async function () { + const driver = new XCUITestDriver({} as any); + const screen = sinon.stub(driver, 'proxyCommand'); + screen + .withArgs('/wda/screen', 'GET') + .onFirstCall() + .resolves({ + scale: 2, + statusBarSize: {width: 951, height: 20.5}, + }); + screen + .withArgs('/wda/screen', 'GET') + .onSecondCall() + .resolves({ + scale: 3, + statusBarSize: {width: 466, height: 30}, + }); + sinon.stub(driver, 'getWindowRect').resolves({x: 0, y: 0, width: 951, height: 669}); + assert.deepEqual(await driver.getViewportRect(), {left: 0, top: 41, width: 1902, height: 1297}); + sinon.assert.calledOnce(screen); + }); + }); }); diff --git a/test/unit/commands/screenshots.spec.ts b/test/unit/commands/screenshots.spec.ts index 7e3d4a7dec..d53c2c8be9 100644 --- a/test/unit/commands/screenshots.spec.ts +++ b/test/unit/commands/screenshots.spec.ts @@ -2,6 +2,7 @@ import assert from 'node:assert/strict'; import {describe, it, beforeEach, afterEach} from 'node:test'; import {errors} from 'appium/driver.js'; +import sharp from 'sharp'; import sinon from 'sinon'; import {XCUITestDriver} from '../../../lib/driver.js'; @@ -95,4 +96,80 @@ describe('screenshots commands', function () { }); }); }); + describe('getViewportScreenshot', function () { + async function screenshot(width: number, height: number): Promise { + return ( + await sharp({create: {width, height, channels: 3, background: '#abcdef'}}) + .png() + .toBuffer() + ).toString('base64'); + } + + it('should crop each display using fresh geometry and the captured image bounds', async function () { + // The image is one pixel larger than the logical window scaled by two. + // Cropping must retain the right and bottom edges, including in a smaller app window. + const capture = sinon.stub(driver, 'getScreenshot'); + sinon.stub(driver, 'getWindowRect').resolves({x: 0, y: 0, width: 400, height: 600}); + for (const [width, height, scale, bar] of [ + [1398, 2034, 3, 20], + [1903, 1339, 2, 24.5], + [1903, 1339, 2, 0], + [1398, 2034, 3, 24], + ]) { + const original = await screenshot(width, height); + capture.resolves(original); + proxyStub.withArgs('/wda/screen', 'GET').resolves({scale, statusBarSize: {width: width / scale, height: bar}}); + proxyStub.resetHistory(); + const result = await driver.getViewportScreenshot(); + const expected = await sharp(Buffer.from(original, 'base64')) + .extract({left: 0, top: Math.trunc(bar * scale), width, height: height - Math.trunc(bar * scale)}) + .raw() + .toBuffer(); + assert.deepEqual(await sharp(Buffer.from(result, 'base64')).raw().toBuffer(), expected); + const dimensions = await sharp(Buffer.from(result, 'base64')).metadata(); + assert.equal(dimensions.width, width); + assert.equal(dimensions.height, height - Math.trunc(bar * scale)); + if (!bar) { + assert.equal(result, original); + } + sinon.assert.calledOnce(proxyStub); + } + }); + + it('should return the original screenshot when status bar geometry cannot be cropped', async function () { + // Use a distinct encoding so an accidental full-image crop/re-encode + // cannot pass the byte-for-byte preservation assertion. + const original = ( + await sharp(Buffer.from(await screenshot(10, 10), 'base64')) + .png({compressionLevel: 0}) + .toBuffer() + ).toString('base64'); + sinon.stub(driver, 'getScreenshot').resolves(original); + for (const [scale, bar] of [ + [2, 5], + [2, 6], + [2, -1], + [2, -0.1], + [0.5, -1], + [2, NaN], + [Infinity, 1], + ]) { + proxyStub.resolves({scale, statusBarSize: {width: 5, height: bar}}); + assert.equal(await driver.getViewportScreenshot(), original); + } + }); + + it('should return web viewport screenshots without requesting native geometry', async function () { + sinon.stub(driver, 'isWebContext').returns(true); + const capture = sinon.stub(driver, '_webExecutionBackend').get(() => ({ + screenshot: async () => 'web screenshot', + })); + try { + assert.equal(await driver.getViewportScreenshot(), 'web screenshot'); + sinon.assert.notCalled(proxyStub); + } finally { + capture.restore(); + } + }); + }); });