From ce34bf21017f71360c8361ab9ae903826b56eb37 Mon Sep 17 00:00:00 2001 From: Jason Steinberg Date: Fri, 25 Sep 2026 11:19:41 -0700 Subject: [PATCH 01/29] feat: add currentDisplayId setting for screenshots --- WebDriverAgentLib/Utilities/FBConfiguration.h | 6 ++ WebDriverAgentLib/Utilities/FBConfiguration.m | 1 + WebDriverAgentLib/Utilities/FBScreen.h | 17 ++++++ WebDriverAgentLib/Utilities/FBScreen.m | 27 +++++++++ WebDriverAgentLib/Utilities/FBScreenshot.h | 4 +- WebDriverAgentLib/Utilities/FBScreenshot.m | 8 ++- WebDriverAgentLib/Utilities/FBSettings.h | 1 + WebDriverAgentLib/Utilities/FBSettings.m | 1 + .../Utilities/FBSettingsHandler.m | 24 ++++++++ .../IntegrationTests/FBScreenTests.m | 60 +++++++++++++++++++ test/functional/helpers/display-settings.ts | 48 +++++++++++++++ test/functional/webdriveragent-e2e.spec.ts | 4 ++ 12 files changed, 197 insertions(+), 4 deletions(-) create mode 100644 test/functional/helpers/display-settings.ts diff --git a/WebDriverAgentLib/Utilities/FBConfiguration.h b/WebDriverAgentLib/Utilities/FBConfiguration.h index 08b755b955..d905c2be1e 100644 --- a/WebDriverAgentLib/Utilities/FBConfiguration.h +++ b/WebDriverAgentLib/Utilities/FBConfiguration.h @@ -368,6 +368,12 @@ typedef NS_ENUM(NSInteger, FBConfigurationKeyboardPreference) { */ @property (atomic, assign) BOOL enforceCustomSnapshots; +/** + * The identifier of the display targeted by screenshots. + * nil means the main display. Reset to nil for each new session. + */ +@property (atomic, copy, nullable) NSNumber *currentDisplayId; + @end NS_ASSUME_NONNULL_END diff --git a/WebDriverAgentLib/Utilities/FBConfiguration.m b/WebDriverAgentLib/Utilities/FBConfiguration.m index 4d788c2e44..26c1524a5f 100644 --- a/WebDriverAgentLib/Utilities/FBConfiguration.m +++ b/WebDriverAgentLib/Utilities/FBConfiguration.m @@ -357,6 +357,7 @@ - (void)resetSessionSettings FBSetCustomParameterForElementSnapshot(FBSnapshotMaxChildrenKey, @INT_MAX); self.useClearTextShortcut = YES; self.limitXpathContextScope = YES; + self.currentDisplayId = nil; #if !TARGET_OS_TV && !TARGET_OS_WATCH self.screenshotOrientationStorage = UIInterfaceOrientationUnknown; #endif diff --git a/WebDriverAgentLib/Utilities/FBScreen.h b/WebDriverAgentLib/Utilities/FBScreen.h index 74e88013e2..4fe17961cd 100644 --- a/WebDriverAgentLib/Utilities/FBScreen.h +++ b/WebDriverAgentLib/Utilities/FBScreen.h @@ -17,6 +17,23 @@ NS_ASSUME_NONNULL_BEGIN */ + (nullable NSArray *> *)screensWithError:(NSError **)error; +/** + The display with the given identifier + + @param displayID The display identifier, as returned by screensWithError: + @param error Set if the display is not available + @return The matching display or nil + */ ++ (nullable XCUIScreen *)screenWithDisplayID:(long long)displayID error:(NSError **)error; + +/** + The display selected by the currentDisplayId setting, or the main display if the setting is not set + + @param error Set if the selected display is no longer available + @return The selected display or nil + */ ++ (nullable XCUIScreen *)currentScreenWithError:(NSError **)error; + /** The identifier of the main device's display */ diff --git a/WebDriverAgentLib/Utilities/FBScreen.m b/WebDriverAgentLib/Utilities/FBScreen.m index f2963002a7..02104961df 100644 --- a/WebDriverAgentLib/Utilities/FBScreen.m +++ b/WebDriverAgentLib/Utilities/FBScreen.m @@ -7,6 +7,8 @@ */ #import "FBScreen.h" +#import "FBConfiguration.h" +#import "FBErrorBuilder.h" #import "XCUIElement+FBIsVisible.h" #import "FBXCodeCompatibility.h" #import "XCUIDevice.h" @@ -40,6 +42,31 @@ @implementation FBScreen return result.copy; } ++ (nullable XCUIScreen *)screenWithDisplayID:(long long)displayID error:(NSError **)error +{ + NSArray *screens = [XCUIDevice.sharedDevice screensOrError:error]; + if (nil == screens) { + return nil; + } + for (XCUIScreen *screen in screens) { + if (screen.displayID == displayID) { + return screen; + } + } + [[FBErrorBuilder.builder withDescriptionFormat:@"No display with id %lld is available. Call /wda/screens to list the available displays", displayID] + buildError:error]; + return nil; +} + ++ (nullable XCUIScreen *)currentScreenWithError:(NSError **)error +{ + NSNumber *displayID = FBConfiguration.sharedInstance.currentDisplayId; + if (nil == displayID) { + return XCUIScreen.mainScreen; + } + return [self screenWithDisplayID:displayID.longLongValue error:error]; +} + + (long long)displayID { return XCUIScreen.mainScreen.displayID; diff --git a/WebDriverAgentLib/Utilities/FBScreenshot.h b/WebDriverAgentLib/Utilities/FBScreenshot.h index b0fdfbe712..3604ba29dc 100644 --- a/WebDriverAgentLib/Utilities/FBScreenshot.h +++ b/WebDriverAgentLib/Utilities/FBScreenshot.h @@ -14,7 +14,7 @@ NS_ASSUME_NONNULL_BEGIN @interface FBScreenshot : NSObject /** - Retrieves non-scaled screenshot of the whole screen + Retrieves non-scaled screenshot of the display selected by the currentDisplayId setting @param quality The number in range 0-3, where 0 is PNG (lossless), 3 is HEIC (lossless), 1- low quality JPEG and 2 - high quality JPEG @param error If there is an error, upon return contains an NSError object that describes the problem. @@ -24,7 +24,7 @@ NS_ASSUME_NONNULL_BEGIN error:(NSError **)error; /** - Retrieves non-scaled screenshot of the whole screen + Retrieves non-scaled screenshot of the display selected by the currentDisplayId setting @param screenID The screen identifier to take the screenshot from @param compressionQuality Normalized screenshot quality value in range 0..1, where 1 is the best quality diff --git a/WebDriverAgentLib/Utilities/FBScreenshot.m b/WebDriverAgentLib/Utilities/FBScreenshot.m index ccdea1bf00..ee37fa332d 100644 --- a/WebDriverAgentLib/Utilities/FBScreenshot.m +++ b/WebDriverAgentLib/Utilities/FBScreenshot.m @@ -15,6 +15,7 @@ #import "FBImageProcessor.h" #import "FBLogger.h" #import "FBMacros.h" +#import "FBScreen.h" #import "FBXCodeCompatibility.h" #import "FBXCTestDaemonsProxy.h" #import "XCTMessagingChannel_RunnerToDaemon-Protocol.h" @@ -60,8 +61,11 @@ + (UTType *)imageUtiWithQuality:(NSUInteger)quality + (NSData *)takeInOriginalResolutionWithQuality:(NSUInteger)quality error:(NSError **)error { - XCUIScreen *mainScreen = XCUIScreen.mainScreen; - return [self.class takeWithScreenID:mainScreen.displayID + XCUIScreen *screen = [FBScreen currentScreenWithError:error]; + if (nil == screen) { + return nil; + } + return [self.class takeWithScreenID:screen.displayID scale:SCREENSHOT_SCALE compressionQuality:[self.class compressionQualityWithQuality:quality] sourceUTI:[self.class imageUtiWithQuality:quality] diff --git a/WebDriverAgentLib/Utilities/FBSettings.h b/WebDriverAgentLib/Utilities/FBSettings.h index 92a436db79..178f299bb6 100644 --- a/WebDriverAgentLib/Utilities/FBSettings.h +++ b/WebDriverAgentLib/Utilities/FBSettings.h @@ -46,5 +46,6 @@ extern NSString *const FB_SETTING_INCLUDE_NATIVE_ACCESSIBILITY_ELEMENT_IN_PAGE_S extern NSString *const FB_SETTING_INCLUDE_MIN_MAX_VALUE_IN_PAGE_SOURCE; extern NSString *const FB_SETTING_INCLUDE_CUSTOM_ACTIONS_IN_PAGE_SOURCE; extern NSString *const FB_SETTING_ENFORCE_CUSTOM_SNAPSHOTS; +extern NSString *const FB_SETTING_CURRENT_DISPLAY_ID; NS_ASSUME_NONNULL_END diff --git a/WebDriverAgentLib/Utilities/FBSettings.m b/WebDriverAgentLib/Utilities/FBSettings.m index 826a226517..aacca26a17 100644 --- a/WebDriverAgentLib/Utilities/FBSettings.m +++ b/WebDriverAgentLib/Utilities/FBSettings.m @@ -42,3 +42,4 @@ NSString* const FB_SETTING_INCLUDE_MIN_MAX_VALUE_IN_PAGE_SOURCE = @"includeMinMaxValueInPageSource"; NSString* const FB_SETTING_INCLUDE_CUSTOM_ACTIONS_IN_PAGE_SOURCE = @"includeCustomActionsInPageSource"; NSString* const FB_SETTING_ENFORCE_CUSTOM_SNAPSHOTS = @"enforceCustomSnapshots"; +NSString* const FB_SETTING_CURRENT_DISPLAY_ID = @"currentDisplayId"; diff --git a/WebDriverAgentLib/Utilities/FBSettingsHandler.m b/WebDriverAgentLib/Utilities/FBSettingsHandler.m index 6184cab5b9..8ff85dcbbd 100644 --- a/WebDriverAgentLib/Utilities/FBSettingsHandler.m +++ b/WebDriverAgentLib/Utilities/FBSettingsHandler.m @@ -12,6 +12,7 @@ #import "FBClassChainQueryParser.h" #import "FBCommandStatus.h" #import "FBConfiguration.h" +#import "FBScreen.h" #import "FBSession.h" #import "FBSettings.h" @@ -33,6 +34,7 @@ static id FBNormalizedSettingValue(id value) FB_SETTING_ACCEPT_ALERT_BUTTON_SELECTOR, FB_SETTING_DISMISS_ALERT_BUTTON_SELECTOR, FB_SETTING_AUTO_CLICK_ALERT_SELECTOR, + FB_SETTING_CURRENT_DISPLAY_ID, nil]; }); return keys; @@ -190,6 +192,25 @@ @implementation FBSettingsHandler FBConfiguration.sharedInstance.limitXpathContextScope = [value boolValue]; return nil; }; + map[FB_SETTING_CURRENT_DISPLAY_ID] = ^FBCommandStatus *(FBSession *session, id value) { + if (nil == value) { + FBConfiguration.sharedInstance.currentDisplayId = nil; + return nil; + } + if (![value isKindOfClass:NSNumber.class] + || CFGetTypeID((__bridge CFTypeRef)value) == CFBooleanGetTypeID() + || [value compare:@([value longLongValue])] != NSOrderedSame) { + return [FBCommandStatus invalidArgumentErrorWithMessage:@"currentDisplayId must be an integer display id returned by /wda/screens" + traceback:nil]; + } + NSError *error; + if (nil == [FBScreen screenWithDisplayID:[value longLongValue] error:&error]) { + return [FBCommandStatus invalidArgumentErrorWithMessage:error.localizedDescription + traceback:nil]; + } + FBConfiguration.sharedInstance.currentDisplayId = @([value longLongValue]); + return nil; + }; #if !TARGET_OS_TV && !TARGET_OS_WATCH map[FB_SETTING_SCREENSHOT_ORIENTATION] = ^FBCommandStatus *(FBSession *session, id value) { NSError *error; @@ -310,6 +331,9 @@ @implementation FBSettingsHandler map[FB_SETTING_LIMIT_XPATH_CONTEXT_SCOPE] = ^id(FBSession *session) { return @(FBConfiguration.sharedInstance.limitXpathContextScope); }; + map[FB_SETTING_CURRENT_DISPLAY_ID] = ^id(FBSession *session) { + return FBConfiguration.sharedInstance.currentDisplayId ?: @([FBScreen displayID]); + }; #if !TARGET_OS_TV && !TARGET_OS_WATCH map[FB_SETTING_SCREENSHOT_ORIENTATION] = ^id(FBSession *session) { return FBConfiguration.sharedInstance.humanReadableScreenshotOrientation; diff --git a/WebDriverAgentTests/IntegrationTests/FBScreenTests.m b/WebDriverAgentTests/IntegrationTests/FBScreenTests.m index a2b6437b75..18a4215e5a 100644 --- a/WebDriverAgentTests/IntegrationTests/FBScreenTests.m +++ b/WebDriverAgentTests/IntegrationTests/FBScreenTests.m @@ -9,7 +9,10 @@ #import #import "FBIntegrationTestCase.h" +#import "FBConfiguration.h" #import "FBScreen.h" +#import "FBScreenshot.h" +#import "XCUIScreen.h" @interface FBScreenTests : FBIntegrationTestCase @end @@ -22,6 +25,12 @@ - (void)setUp [self launchApplication]; } +- (void)tearDown +{ + FBConfiguration.sharedInstance.currentDisplayId = nil; + [super tearDown]; +} + - (void)testDisplayID { XCTAssertGreaterThanOrEqual([FBScreen displayID], 0LL); @@ -50,6 +59,57 @@ - (void)testScreens XCTAssertNotNil(mainScreen[@"traits"]); } +- (void)testScreenWithDisplayID +{ + NSError *error = nil; + XCUIScreen *screen = [FBScreen screenWithDisplayID:[FBScreen displayID] error:&error]; + XCTAssertNil(error); + XCTAssertEqual(screen.displayID, [FBScreen displayID]); + + XCTAssertNil([FBScreen screenWithDisplayID:[self unknownDisplayID] error:&error]); + XCTAssertNotNil(error); +} + +- (void)testCurrentScreenDefaultsToMainScreen +{ + NSError *error = nil; + XCTAssertTrue([FBScreen currentScreenWithError:&error].isMainScreen); + XCTAssertNil(error); +} + +- (void)testCurrentScreenFollowsSetting +{ + FBConfiguration.sharedInstance.currentDisplayId = @([FBScreen displayID]); + NSError *error = nil; + XCTAssertEqual([FBScreen currentScreenWithError:&error].displayID, [FBScreen displayID]); + XCTAssertNil(error); + XCTAssertNotNil([FBScreenshot takeInOriginalResolutionWithQuality:0 error:&error]); + XCTAssertNil(error); + + FBConfiguration.sharedInstance.currentDisplayId = @([self unknownDisplayID]); + XCTAssertNil([FBScreen currentScreenWithError:&error]); + XCTAssertNotNil(error); + error = nil; + XCTAssertNil([FBScreenshot takeInOriginalResolutionWithQuality:0 error:&error]); + XCTAssertNotNil(error); +} + +- (void)testSessionResetClearsCurrentDisplay +{ + FBConfiguration.sharedInstance.currentDisplayId = @([FBScreen displayID]); + [FBConfiguration.sharedInstance resetSessionSettings]; + XCTAssertNil(FBConfiguration.sharedInstance.currentDisplayId); +} + +- (long long)unknownDisplayID +{ + long long maxID = 0; + for (NSDictionary *screen in [FBScreen screensWithError:nil]) { + maxID = MAX(maxID, [screen[@"displayId"] longLongValue]); + } + return maxID + 1; +} + - (void)testScreenScale { XCTAssertTrue([FBScreen scale] >= 2); diff --git a/test/functional/helpers/display-settings.ts b/test/functional/helpers/display-settings.ts new file mode 100644 index 0000000000..bd62dc57f4 --- /dev/null +++ b/test/functional/helpers/display-settings.ts @@ -0,0 +1,48 @@ +import assert from 'node:assert/strict'; + +/** Exercise the currentDisplayId setting against a live WDA session. */ +export async function assertCurrentDisplayIdSetting(baseUrl: string, sessionId: string): Promise { + const screensResponse = await fetch(`${baseUrl}/wda/screens`); + assert.equal(screensResponse.status, 200); + const {value: screens} = (await screensResponse.json()) as { + value: {displayId: number; isMain: boolean}[]; + }; + const main = screens.find((screen) => screen.isMain); + assert.ok(main, 'expected a main display'); + const sessionUrl = `${baseUrl}/session/${sessionId}`; + const setDisplay = async (value: unknown) => + await fetch(`${sessionUrl}/appium/settings`, { + method: 'POST', + headers: {'Content-Type': 'application/json'}, + body: JSON.stringify({settings: {currentDisplayId: value}}), + }); + const getDisplay = async () => { + const response = await fetch(`${sessionUrl}/appium/settings`); + assert.equal(response.status, 200); + return ((await response.json()) as {value: {currentDisplayId: number}}).value.currentDisplayId; + }; + const assertScreenshot = async () => { + const response = await fetch(`${sessionUrl}/screenshot`); + assert.equal(response.status, 200); + const {value} = (await response.json()) as {value: string}; + assert.ok(Buffer.from(value, 'base64').length > 0, 'expected screenshot data'); + }; + + assert.equal(await getDisplay(), main.displayId); + assert.equal((await setDisplay(main.displayId)).status, 200); + assert.equal(await getDisplay(), main.displayId); + await assertScreenshot(); + + const unknownId = Math.max(...screens.map((screen) => screen.displayId)) + 1; + for (const value of [unknownId, 1.5, '1', true]) { + const response = await setDisplay(value); + assert.equal(response.status, 400, `expected invalid argument for ${JSON.stringify(value)}`); + const {value: body} = (await response.json()) as {value: {error: string}}; + assert.equal(body.error, 'invalid argument'); + } + assert.equal(await getDisplay(), main.displayId); + + assert.equal((await setDisplay(null)).status, 200); + assert.equal(await getDisplay(), main.displayId); + await assertScreenshot(); +} diff --git a/test/functional/webdriveragent-e2e.spec.ts b/test/functional/webdriveragent-e2e.spec.ts index e02dc7a62a..5f6f32b58b 100644 --- a/test/functional/webdriveragent-e2e.spec.ts +++ b/test/functional/webdriveragent-e2e.spec.ts @@ -5,6 +5,7 @@ import {describe, before, afterEach, it} from 'node:test'; import type {AppleDevice} from '../../lib/types.js'; import {WebDriverAgent} from '../../lib/webdriveragent.js'; import {PLATFORM_NAME, PLATFORM_VERSION} from './desired.js'; +import {assertCurrentDisplayIdSetting} from './helpers/display-settings.js'; import {getTargetDevice} from './helpers/simulator.js'; const WDA_BASE_URL = 'http://localhost:8100'; @@ -59,6 +60,9 @@ async function assertSessionScopedSourceWorks(): Promise { assert.equal(sessionSourceResponse.status, 200); const sessionSource = (await sessionSourceResponse.json()) as {value: string}; assert.ok(sessionSource.value.length > 0, 'expected session-scoped /source to return non-empty page source'); + if (PLATFORM_NAME.toLowerCase() === 'ios') { + await assertCurrentDisplayIdSetting(WDA_BASE_URL, wdaSessionId); + } } finally { await fetch(`${WDA_BASE_URL}/session/${wdaSessionId}`, {method: 'DELETE'}); } From d8cf1b800a0fd7a89ac5c8633ee72611880cd217 Mon Sep 17 00:00:00 2001 From: Jason Steinberg Date: Fri, 25 Sep 2026 15:52:08 -0700 Subject: [PATCH 02/29] chore: remove boolean check from currentDisplayId validation --- WebDriverAgentLib/Utilities/FBSettingsHandler.m | 1 - test/functional/helpers/display-settings.ts | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/WebDriverAgentLib/Utilities/FBSettingsHandler.m b/WebDriverAgentLib/Utilities/FBSettingsHandler.m index 8ff85dcbbd..91eb935a31 100644 --- a/WebDriverAgentLib/Utilities/FBSettingsHandler.m +++ b/WebDriverAgentLib/Utilities/FBSettingsHandler.m @@ -198,7 +198,6 @@ @implementation FBSettingsHandler return nil; } if (![value isKindOfClass:NSNumber.class] - || CFGetTypeID((__bridge CFTypeRef)value) == CFBooleanGetTypeID() || [value compare:@([value longLongValue])] != NSOrderedSame) { return [FBCommandStatus invalidArgumentErrorWithMessage:@"currentDisplayId must be an integer display id returned by /wda/screens" traceback:nil]; diff --git a/test/functional/helpers/display-settings.ts b/test/functional/helpers/display-settings.ts index bd62dc57f4..c5c37086b4 100644 --- a/test/functional/helpers/display-settings.ts +++ b/test/functional/helpers/display-settings.ts @@ -34,7 +34,7 @@ export async function assertCurrentDisplayIdSetting(baseUrl: string, sessionId: await assertScreenshot(); const unknownId = Math.max(...screens.map((screen) => screen.displayId)) + 1; - for (const value of [unknownId, 1.5, '1', true]) { + for (const value of [unknownId, 1.5, '1']) { const response = await setDisplay(value); assert.equal(response.status, 400, `expected invalid argument for ${JSON.stringify(value)}`); const {value: body} = (await response.json()) as {value: {error: string}}; From 35d9e85c2daaba0a6245d4550e64f2228646bfe1 Mon Sep 17 00:00:00 2001 From: Jason Steinberg Date: Fri, 25 Sep 2026 11:19:41 -0700 Subject: [PATCH 03/29] feat: target the currentDisplayId display with W3C actions --- WebDriverAgentLib/Utilities/FBMathUtils.h | 8 ++ WebDriverAgentLib/Utilities/FBMathUtils.m | 17 ++++ .../Utilities/FBW3CActionsSynthesizer.m | 37 +++++++- .../FBW3CTouchActionsIntegrationTests.m | 93 +++++++++++++++++++ .../UnitTests/FBMathUtilsTests.m | 29 ++++++ 5 files changed, 181 insertions(+), 3 deletions(-) diff --git a/WebDriverAgentLib/Utilities/FBMathUtils.h b/WebDriverAgentLib/Utilities/FBMathUtils.h index ac3a3423d2..56392db7d9 100644 --- a/WebDriverAgentLib/Utilities/FBMathUtils.h +++ b/WebDriverAgentLib/Utilities/FBMathUtils.h @@ -37,6 +37,14 @@ BOOL FBRectFuzzyEqualToRect(CGRect rect1, CGRect rect2, CGFloat threshold); #if !TARGET_OS_TV && !TARGET_OS_WATCH /*! Inverts size if necessary to match current screen orientation */ CGSize FBAdjustDimensionsForApplication(CGSize actualSize, UIInterfaceOrientation orientation); + +/*! + Offsets XCTest screenPoint coordinates for an explicitly selected display. + XCTest rotates these coordinates using the main screen's dimensions even when + the synthesized event record targets another display. Sizes are in points. + */ +CGPoint FBDisplayCoordinateOffset(CGSize mainSize, CGSize displaySize, + UIInterfaceOrientation orientation); #endif #if !TARGET_OS_TV diff --git a/WebDriverAgentLib/Utilities/FBMathUtils.m b/WebDriverAgentLib/Utilities/FBMathUtils.m index 5ae4bb269c..d500c1a4e0 100644 --- a/WebDriverAgentLib/Utilities/FBMathUtils.m +++ b/WebDriverAgentLib/Utilities/FBMathUtils.m @@ -107,3 +107,20 @@ BOOL FBScrollGestureOffsets(CGRect scrollingFrame, (endPoint.y - anchorFrame.origin.y) / anchorFrame.size.height); return YES; } + +#if !TARGET_OS_TV && !TARGET_OS_WATCH +CGPoint FBDisplayCoordinateOffset(CGSize mainSize, CGSize displaySize, + UIInterfaceOrientation orientation) +{ + CGPoint offset = CGPointZero; + if (orientation == UIInterfaceOrientationLandscapeRight + || orientation == UIInterfaceOrientationPortraitUpsideDown) { + offset.x = displaySize.width - mainSize.width; + } + if (orientation == UIInterfaceOrientationLandscapeLeft + || orientation == UIInterfaceOrientationPortraitUpsideDown) { + offset.y = displaySize.height - mainSize.height; + } + return offset; +} +#endif diff --git a/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m b/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m index c37f2a443c..4a1957dafb 100644 --- a/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m +++ b/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m @@ -15,6 +15,7 @@ #import "FBMacros.h" #import "FBMathUtils.h" #import "FBProtocolHelpers.h" +#import "FBScreen.h" #import "FBW3CActionsHelpers.h" #import "FBXCodeCompatibility.h" #import "FBXCTestDaemonsProxy.h" @@ -28,6 +29,7 @@ #import "XCSynthesizedEventRecord.h" #import "XCPointerEventPath.h" #import "XCPointerEvent.h" +#import "XCUIScreen.h" static NSString *const FB_KEY_TYPE = @"type"; @@ -849,9 +851,32 @@ @implementation FBW3CActionsSynthesizer - (nullable XCSynthesizedEventRecord *)synthesizeWithError:(NSError **)error { - XCSynthesizedEventRecord *eventRecord = [[XCSynthesizedEventRecord alloc] - initWithName:@"W3C Touch Action" - interfaceOrientation:self.application.interfaceOrientation]; + XCUIScreen *screen = [FBScreen currentScreenWithError:error]; + if (nil == screen) { + return nil; + } + XCSynthesizedEventRecord *eventRecord; + CGPoint displayCorrection = CGPointZero; + if (screen.isMainScreen) { + eventRecord = [[XCSynthesizedEventRecord alloc] initWithName:@"W3C Touch Action" + interfaceOrientation:self.application.interfaceOrientation]; + } else { + if (![XCSynthesizedEventRecord instancesRespondToSelector:@selector(initWithName:displayID:interfaceOrientation:)]) { + if (error) { + *error = [[FBErrorBuilder.builder + withDescription:@"Actions on a display other than the main one are not supported by this XCTest version"] build]; + } + return nil; + } + eventRecord = [[XCSynthesizedEventRecord alloc] initWithName:@"W3C Touch Action" + displayID:(unsigned long long)screen.displayID + interfaceOrientation:self.application.interfaceOrientation]; + XCUIScreen *mainScreen = XCUIScreen.mainScreen; + displayCorrection = FBDisplayCoordinateOffset( + CGSizeMake(mainScreen.bounds.size.width / mainScreen.scale, mainScreen.bounds.size.height / mainScreen.scale), + CGSizeMake(screen.bounds.size.width / screen.scale, screen.bounds.size.height / screen.scale), + eventRecord.interfaceOrientation); + } NSMutableDictionary *> *actionsMapping = [NSMutableDictionary new]; NSMutableArray *actionIds = [NSMutableArray new]; for (NSDictionary *action in self.actions) { @@ -893,6 +918,12 @@ - (nullable XCSynthesizedEventRecord *)synthesizeWithError:(NSError **)error return nil; } for (XCPointerEventPath *eventPath in eventPaths) { + if (!CGPointEqualToPoint(displayCorrection, CGPointZero)) { + for (XCPointerEvent *pointerEvent in eventPath.pointerEvents) { + CGPoint point = pointerEvent.coordinate; + pointerEvent.coordinate = CGPointMake(point.x + displayCorrection.x, point.y + displayCorrection.y); + } + } [eventRecord addPointerEventPath:eventPath]; } } diff --git a/WebDriverAgentTests/IntegrationTests/FBW3CTouchActionsIntegrationTests.m b/WebDriverAgentTests/IntegrationTests/FBW3CTouchActionsIntegrationTests.m index 8a41983c42..b623af50f3 100644 --- a/WebDriverAgentTests/IntegrationTests/FBW3CTouchActionsIntegrationTests.m +++ b/WebDriverAgentTests/IntegrationTests/FBW3CTouchActionsIntegrationTests.m @@ -9,6 +9,9 @@ #import #import "FBIntegrationTestCase.h" +#import "FBConfiguration.h" +#import "FBMathUtils.h" +#import "FBScreen.h" #import "XCUIElement.h" #import "XCUIDevice.h" @@ -21,6 +24,7 @@ #import "XCSynthesizedEventRecord.h" #import "XCPointerEventPath.h" #import "XCPointerEvent.h" +#import "XCUIScreen.h" @interface FBW3CTouchActionsIntegrationTestsPart1 : FBIntegrationTestCase @end @@ -53,11 +57,100 @@ - (void)setUp - (void)tearDown { + FBConfiguration.sharedInstance.currentDisplayId = nil; [self clearAlert]; [self resetOrientation]; [super tearDown]; } +- (nullable XCSynthesizedEventRecord *)tapRecordWithError:(NSError **)error +{ + NSArray *gesture = @[@{ + @"type": @"pointer", @"id": @"finger", @"parameters": @{@"pointerType": @"touch"}, + @"actions": @[ + @{@"type": @"pointerMove", @"x": @100, @"y": @100, @"duration": @0}, + @{@"type": @"pointerDown", @"button": @0}, + @{@"type": @"pointerUp", @"button": @0}, + ], + }]; + FBW3CActionsSynthesizer *synthesizer = [[FBW3CActionsSynthesizer alloc] initWithActions:gesture + forApplication:self.testedApplication + elementCache:nil + error:error]; + return [synthesizer synthesizeWithError:error]; +} + +- (NSArray *)pointerEventsOfRecord:(XCSynthesizedEventRecord *)record +{ + XCPointerEventPath *eventPath = record.eventPaths.firstObject; + return eventPath.pointerEvents; +} + +- (void)testMainDisplaySettingKeepsDefaultCoordinates +{ + for (NSNumber *orientation in @[@(UIDeviceOrientationPortrait), + @(UIDeviceOrientationLandscapeLeft), + @(UIDeviceOrientationLandscapeRight)]) { + [[XCUIDevice sharedDevice] fb_setDeviceInterfaceOrientation:orientation.integerValue]; + NSError *error = nil; + FBConfiguration.sharedInstance.currentDisplayId = nil; + XCSynthesizedEventRecord *baseline = [self tapRecordWithError:&error]; + FBConfiguration.sharedInstance.currentDisplayId = @([FBScreen displayID]); + XCSynthesizedEventRecord *selected = [self tapRecordWithError:&error]; + XCTAssertNotNil(baseline); + XCTAssertNotNil(selected); + XCTAssertNil(error); + NSArray *baselineEvents = [self pointerEventsOfRecord:baseline]; + NSArray *selectedEvents = [self pointerEventsOfRecord:selected]; + XCTAssertEqual(baselineEvents.count, selectedEvents.count); + for (NSUInteger i = 0; i < baselineEvents.count; i++) { + XCTAssertTrue(CGPointEqualToPoint(baselineEvents[i].coordinate, selectedEvents[i].coordinate)); + } + } +} + +- (void)testUnavailableDisplaySettingIsRejected +{ + long long maxID = 0; + for (NSDictionary *screen in [FBScreen screensWithError:nil]) { + maxID = MAX(maxID, [screen[@"displayId"] longLongValue]); + } + FBConfiguration.sharedInstance.currentDisplayId = @(maxID + 1); + NSError *error = nil; + XCTAssertNil([self tapRecordWithError:&error]); + XCTAssertNotNil(error); +} + +- (void)testSecondaryDisplaySetting +{ + XCUIScreen *secondary = nil; + for (XCUIScreen *screen in [XCUIDevice.sharedDevice screensOrError:nil]) { + if (!screen.isMainScreen) { + secondary = screen; + break; + } + } + if (nil == secondary) { + XCTSkip(@"The device has no secondary display"); + } + NSError *error = nil; + XCSynthesizedEventRecord *baseline = [self tapRecordWithError:&error]; + FBConfiguration.sharedInstance.currentDisplayId = @(secondary.displayID); + XCSynthesizedEventRecord *selected = [self tapRecordWithError:&error]; + XCTAssertNotNil(selected); + XCTAssertNil(error); + XCTAssertEqual(selected.displayID, (unsigned long long)secondary.displayID); + XCUIScreen *main = XCUIScreen.mainScreen; + CGPoint offset = FBDisplayCoordinateOffset( + CGSizeMake(main.bounds.size.width / main.scale, main.bounds.size.height / main.scale), + CGSizeMake(secondary.bounds.size.width / secondary.scale, secondary.bounds.size.height / secondary.scale), + selected.interfaceOrientation); + CGPoint expected = [self pointerEventsOfRecord:baseline].firstObject.coordinate; + CGPoint actual = [self pointerEventsOfRecord:selected].firstObject.coordinate; + XCTAssertEqual(actual.x, expected.x + offset.x); + XCTAssertEqual(actual.y, expected.y + offset.y); +} + - (void)testErroneousGestures { NSArray *> *> *invalidGestures = diff --git a/WebDriverAgentTests/UnitTests/FBMathUtilsTests.m b/WebDriverAgentTests/UnitTests/FBMathUtilsTests.m index 1e937e3ce0..b8f8d4acc5 100644 --- a/WebDriverAgentTests/UnitTests/FBMathUtilsTests.m +++ b/WebDriverAgentTests/UnitTests/FBMathUtilsTests.m @@ -148,4 +148,33 @@ - (void)testScrollGestureOffsetsWithEmptyFrame XCTAssertTrue(endOffset.dx == -1 && endOffset.dy == -1); } +- (void)testDisplayCoordinateOffsets +{ + CGSize mainSize = CGSizeMake(402, 874); + CGSize innerSize = CGSizeMake(669, 951); + NSArray *orientations = @[ + @(UIInterfaceOrientationPortrait), + @(UIInterfaceOrientationLandscapeLeft), + @(UIInterfaceOrientationLandscapeRight), + @(UIInterfaceOrientationPortraitUpsideDown), + ]; + NSArray *expected = @[ + [NSValue valueWithCGPoint:CGPointZero], + [NSValue valueWithCGPoint:CGPointMake(0, 77)], + [NSValue valueWithCGPoint:CGPointMake(267, 0)], + [NSValue valueWithCGPoint:CGPointMake(267, 77)], + ]; + for (NSUInteger i = 0; i < orientations.count; i++) { + UIInterfaceOrientation orientation = orientations[i].integerValue; + XCTAssertTrue(CGPointEqualToPoint( + FBDisplayCoordinateOffset(mainSize, innerSize, orientation), + expected[i].CGPointValue)); + XCTAssertTrue(CGPointEqualToPoint( + FBDisplayCoordinateOffset(mainSize, mainSize, orientation), CGPointZero)); + CGPoint reverse = FBDisplayCoordinateOffset(innerSize, mainSize, orientation); + XCTAssertEqual(reverse.x, -expected[i].CGPointValue.x); + XCTAssertEqual(reverse.y, -expected[i].CGPointValue.y); + } +} + @end From 4f152bc42f5412c189f9d78dbf0a4a1a47bbae85 Mon Sep 17 00:00:00 2001 From: Jason Steinberg Date: Mon, 28 Sep 2026 15:15:34 -0700 Subject: [PATCH 04/29] fix: validate display settings and report available ids --- WebDriverAgentLib/Utilities/FBScreen.m | 7 ++++++- WebDriverAgentLib/Utilities/FBSettingsHandler.m | 1 + WebDriverAgentTests/IntegrationTests/FBScreenTests.m | 6 ++++++ test/functional/helpers/display-settings.ts | 3 ++- 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/WebDriverAgentLib/Utilities/FBScreen.m b/WebDriverAgentLib/Utilities/FBScreen.m index 02104961df..16e2b36588 100644 --- a/WebDriverAgentLib/Utilities/FBScreen.m +++ b/WebDriverAgentLib/Utilities/FBScreen.m @@ -53,7 +53,12 @@ + (nullable XCUIScreen *)screenWithDisplayID:(long long)displayID error:(NSError return screen; } } - [[FBErrorBuilder.builder withDescriptionFormat:@"No display with id %lld is available. Call /wda/screens to list the available displays", displayID] + NSMutableArray *availableIDs = [NSMutableArray arrayWithCapacity:screens.count]; + for (XCUIScreen *screen in screens) { + [availableIDs addObject:@(screen.displayID)]; + } + [[FBErrorBuilder.builder withDescriptionFormat:@"No display with id %lld is available. Available display ids: [%@]", + displayID, [availableIDs componentsJoinedByString:@", "]] buildError:error]; return nil; } diff --git a/WebDriverAgentLib/Utilities/FBSettingsHandler.m b/WebDriverAgentLib/Utilities/FBSettingsHandler.m index 91eb935a31..8ff85dcbbd 100644 --- a/WebDriverAgentLib/Utilities/FBSettingsHandler.m +++ b/WebDriverAgentLib/Utilities/FBSettingsHandler.m @@ -198,6 +198,7 @@ @implementation FBSettingsHandler return nil; } if (![value isKindOfClass:NSNumber.class] + || CFGetTypeID((__bridge CFTypeRef)value) == CFBooleanGetTypeID() || [value compare:@([value longLongValue])] != NSOrderedSame) { return [FBCommandStatus invalidArgumentErrorWithMessage:@"currentDisplayId must be an integer display id returned by /wda/screens" traceback:nil]; diff --git a/WebDriverAgentTests/IntegrationTests/FBScreenTests.m b/WebDriverAgentTests/IntegrationTests/FBScreenTests.m index 18a4215e5a..37c96c5361 100644 --- a/WebDriverAgentTests/IntegrationTests/FBScreenTests.m +++ b/WebDriverAgentTests/IntegrationTests/FBScreenTests.m @@ -68,6 +68,12 @@ - (void)testScreenWithDisplayID XCTAssertNil([FBScreen screenWithDisplayID:[self unknownDisplayID] error:&error]); XCTAssertNotNil(error); + NSMutableArray *availableIDs = [NSMutableArray array]; + for (NSDictionary *availableScreen in [FBScreen screensWithError:nil]) { + [availableIDs addObject:availableScreen[@"displayId"]]; + } + XCTAssertTrue([error.localizedDescription containsString: + [NSString stringWithFormat:@"Available display ids: [%@]", [availableIDs componentsJoinedByString:@", "]]]); } - (void)testCurrentScreenDefaultsToMainScreen diff --git a/test/functional/helpers/display-settings.ts b/test/functional/helpers/display-settings.ts index c5c37086b4..f1107e2784 100644 --- a/test/functional/helpers/display-settings.ts +++ b/test/functional/helpers/display-settings.ts @@ -34,11 +34,12 @@ export async function assertCurrentDisplayIdSetting(baseUrl: string, sessionId: await assertScreenshot(); const unknownId = Math.max(...screens.map((screen) => screen.displayId)) + 1; - for (const value of [unknownId, 1.5, '1']) { + for (const value of [true, false, unknownId, 1.5, '1']) { const response = await setDisplay(value); assert.equal(response.status, 400, `expected invalid argument for ${JSON.stringify(value)}`); const {value: body} = (await response.json()) as {value: {error: string}}; assert.equal(body.error, 'invalid argument'); + assert.equal(await getDisplay(), main.displayId, 'invalid input must not change the display'); } assert.equal(await getDisplay(), main.displayId); From e1b1bdaaae835c131d7c303d282162593910aad7 Mon Sep 17 00:00:00 2001 From: Jason Steinberg Date: Mon, 28 Sep 2026 15:22:55 -0700 Subject: [PATCH 05/29] test: fix available-display assertion syntax --- WebDriverAgentTests/IntegrationTests/FBScreenTests.m | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/WebDriverAgentTests/IntegrationTests/FBScreenTests.m b/WebDriverAgentTests/IntegrationTests/FBScreenTests.m index 37c96c5361..012b2614b5 100644 --- a/WebDriverAgentTests/IntegrationTests/FBScreenTests.m +++ b/WebDriverAgentTests/IntegrationTests/FBScreenTests.m @@ -72,8 +72,9 @@ - (void)testScreenWithDisplayID for (NSDictionary *availableScreen in [FBScreen screensWithError:nil]) { [availableIDs addObject:availableScreen[@"displayId"]]; } - XCTAssertTrue([error.localizedDescription containsString: - [NSString stringWithFormat:@"Available display ids: [%@]", [availableIDs componentsJoinedByString:@", "]]]); + NSString *availableDisplays = [NSString stringWithFormat:@"Available display ids: [%@]", + [availableIDs componentsJoinedByString:@", "]]; + XCTAssertTrue([error.localizedDescription containsString:availableDisplays]); } - (void)testCurrentScreenDefaultsToMainScreen From 05636fb43063bf52eb5c0dbaf04ee3508da27afe Mon Sep 17 00:00:00 2001 From: Jason Steinberg Date: Mon, 28 Sep 2026 15:30:54 -0700 Subject: [PATCH 06/29] refactor: clarify the display coordinate correction --- WebDriverAgentLib/Utilities/FBMathUtils.h | 2 +- WebDriverAgentLib/Utilities/FBMathUtils.m | 8 ++++---- .../Utilities/FBW3CActionsSynthesizer.m | 9 ++++++--- .../IntegrationTests/FBScreenTests.m | 5 +++-- .../FBW3CTouchActionsIntegrationTests.m | 6 +++--- .../UnitTests/FBMathUtilsTests.m | 17 +++++++++-------- 6 files changed, 26 insertions(+), 21 deletions(-) diff --git a/WebDriverAgentLib/Utilities/FBMathUtils.h b/WebDriverAgentLib/Utilities/FBMathUtils.h index 56392db7d9..ebce725ebc 100644 --- a/WebDriverAgentLib/Utilities/FBMathUtils.h +++ b/WebDriverAgentLib/Utilities/FBMathUtils.h @@ -43,7 +43,7 @@ CGSize FBAdjustDimensionsForApplication(CGSize actualSize, UIInterfaceOrientatio XCTest rotates these coordinates using the main screen's dimensions even when the synthesized event record targets another display. Sizes are in points. */ -CGPoint FBDisplayCoordinateOffset(CGSize mainSize, CGSize displaySize, +CGVector FBDisplayCoordinateOffset(CGSize mainSize, CGSize displaySize, UIInterfaceOrientation orientation); #endif diff --git a/WebDriverAgentLib/Utilities/FBMathUtils.m b/WebDriverAgentLib/Utilities/FBMathUtils.m index d500c1a4e0..0e1faa477f 100644 --- a/WebDriverAgentLib/Utilities/FBMathUtils.m +++ b/WebDriverAgentLib/Utilities/FBMathUtils.m @@ -109,17 +109,17 @@ BOOL FBScrollGestureOffsets(CGRect scrollingFrame, } #if !TARGET_OS_TV && !TARGET_OS_WATCH -CGPoint FBDisplayCoordinateOffset(CGSize mainSize, CGSize displaySize, +CGVector FBDisplayCoordinateOffset(CGSize mainSize, CGSize displaySize, UIInterfaceOrientation orientation) { - CGPoint offset = CGPointZero; + CGVector offset = CGVectorMake(0, 0); if (orientation == UIInterfaceOrientationLandscapeRight || orientation == UIInterfaceOrientationPortraitUpsideDown) { - offset.x = displaySize.width - mainSize.width; + offset.dx = displaySize.width - mainSize.width; } if (orientation == UIInterfaceOrientationLandscapeLeft || orientation == UIInterfaceOrientationPortraitUpsideDown) { - offset.y = displaySize.height - mainSize.height; + offset.dy = displaySize.height - mainSize.height; } return offset; } diff --git a/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m b/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m index 4a1957dafb..7c3a4f3f5a 100644 --- a/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m +++ b/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m @@ -856,7 +856,7 @@ - (nullable XCSynthesizedEventRecord *)synthesizeWithError:(NSError **)error return nil; } XCSynthesizedEventRecord *eventRecord; - CGPoint displayCorrection = CGPointZero; + CGVector displayCorrection = CGVectorMake(0, 0); if (screen.isMainScreen) { eventRecord = [[XCSynthesizedEventRecord alloc] initWithName:@"W3C Touch Action" interfaceOrientation:self.application.interfaceOrientation]; @@ -871,6 +871,9 @@ - (nullable XCSynthesizedEventRecord *)synthesizeWithError:(NSError **)error eventRecord = [[XCSynthesizedEventRecord alloc] initWithName:@"W3C Touch Action" displayID:(unsigned long long)screen.displayID interfaceOrientation:self.application.interfaceOrientation]; + // XCUICoordinate.screenPoint uses the main display's size when rotating + // coordinates. Selecting an event display does not change that transform; + // compensate for the difference before dispatching to the selected display. XCUIScreen *mainScreen = XCUIScreen.mainScreen; displayCorrection = FBDisplayCoordinateOffset( CGSizeMake(mainScreen.bounds.size.width / mainScreen.scale, mainScreen.bounds.size.height / mainScreen.scale), @@ -918,10 +921,10 @@ - (nullable XCSynthesizedEventRecord *)synthesizeWithError:(NSError **)error return nil; } for (XCPointerEventPath *eventPath in eventPaths) { - if (!CGPointEqualToPoint(displayCorrection, CGPointZero)) { + if (displayCorrection.dx != 0 || displayCorrection.dy != 0) { for (XCPointerEvent *pointerEvent in eventPath.pointerEvents) { CGPoint point = pointerEvent.coordinate; - pointerEvent.coordinate = CGPointMake(point.x + displayCorrection.x, point.y + displayCorrection.y); + pointerEvent.coordinate = CGPointMake(point.x + displayCorrection.dx, point.y + displayCorrection.dy); } } [eventRecord addPointerEventPath:eventPath]; diff --git a/WebDriverAgentTests/IntegrationTests/FBScreenTests.m b/WebDriverAgentTests/IntegrationTests/FBScreenTests.m index 37c96c5361..012b2614b5 100644 --- a/WebDriverAgentTests/IntegrationTests/FBScreenTests.m +++ b/WebDriverAgentTests/IntegrationTests/FBScreenTests.m @@ -72,8 +72,9 @@ - (void)testScreenWithDisplayID for (NSDictionary *availableScreen in [FBScreen screensWithError:nil]) { [availableIDs addObject:availableScreen[@"displayId"]]; } - XCTAssertTrue([error.localizedDescription containsString: - [NSString stringWithFormat:@"Available display ids: [%@]", [availableIDs componentsJoinedByString:@", "]]]); + NSString *availableDisplays = [NSString stringWithFormat:@"Available display ids: [%@]", + [availableIDs componentsJoinedByString:@", "]]; + XCTAssertTrue([error.localizedDescription containsString:availableDisplays]); } - (void)testCurrentScreenDefaultsToMainScreen diff --git a/WebDriverAgentTests/IntegrationTests/FBW3CTouchActionsIntegrationTests.m b/WebDriverAgentTests/IntegrationTests/FBW3CTouchActionsIntegrationTests.m index b623af50f3..57cd13774d 100644 --- a/WebDriverAgentTests/IntegrationTests/FBW3CTouchActionsIntegrationTests.m +++ b/WebDriverAgentTests/IntegrationTests/FBW3CTouchActionsIntegrationTests.m @@ -141,14 +141,14 @@ - (void)testSecondaryDisplaySetting XCTAssertNil(error); XCTAssertEqual(selected.displayID, (unsigned long long)secondary.displayID); XCUIScreen *main = XCUIScreen.mainScreen; - CGPoint offset = FBDisplayCoordinateOffset( + CGVector offset = FBDisplayCoordinateOffset( CGSizeMake(main.bounds.size.width / main.scale, main.bounds.size.height / main.scale), CGSizeMake(secondary.bounds.size.width / secondary.scale, secondary.bounds.size.height / secondary.scale), selected.interfaceOrientation); CGPoint expected = [self pointerEventsOfRecord:baseline].firstObject.coordinate; CGPoint actual = [self pointerEventsOfRecord:selected].firstObject.coordinate; - XCTAssertEqual(actual.x, expected.x + offset.x); - XCTAssertEqual(actual.y, expected.y + offset.y); + XCTAssertEqual(actual.x, expected.x + offset.dx); + XCTAssertEqual(actual.y, expected.y + offset.dy); } - (void)testErroneousGestures diff --git a/WebDriverAgentTests/UnitTests/FBMathUtilsTests.m b/WebDriverAgentTests/UnitTests/FBMathUtilsTests.m index b8f8d4acc5..cfd3486890 100644 --- a/WebDriverAgentTests/UnitTests/FBMathUtilsTests.m +++ b/WebDriverAgentTests/UnitTests/FBMathUtilsTests.m @@ -166,14 +166,15 @@ - (void)testDisplayCoordinateOffsets ]; for (NSUInteger i = 0; i < orientations.count; i++) { UIInterfaceOrientation orientation = orientations[i].integerValue; - XCTAssertTrue(CGPointEqualToPoint( - FBDisplayCoordinateOffset(mainSize, innerSize, orientation), - expected[i].CGPointValue)); - XCTAssertTrue(CGPointEqualToPoint( - FBDisplayCoordinateOffset(mainSize, mainSize, orientation), CGPointZero)); - CGPoint reverse = FBDisplayCoordinateOffset(innerSize, mainSize, orientation); - XCTAssertEqual(reverse.x, -expected[i].CGPointValue.x); - XCTAssertEqual(reverse.y, -expected[i].CGPointValue.y); + CGVector offset = FBDisplayCoordinateOffset(mainSize, innerSize, orientation); + XCTAssertEqual(offset.dx, expected[i].CGPointValue.x); + XCTAssertEqual(offset.dy, expected[i].CGPointValue.y); + CGVector sameDisplay = FBDisplayCoordinateOffset(mainSize, mainSize, orientation); + XCTAssertEqual(sameDisplay.dx, 0); + XCTAssertEqual(sameDisplay.dy, 0); + CGVector reverse = FBDisplayCoordinateOffset(innerSize, mainSize, orientation); + XCTAssertEqual(reverse.dx, -expected[i].CGPointValue.x); + XCTAssertEqual(reverse.dy, -expected[i].CGPointValue.y); } } From 1dce81f71a8215cb68b8207ad78e3c77d0af0501 Mon Sep 17 00:00:00 2001 From: Kazuaki Matsuo Date: Sat, 3 Oct 2026 00:01:08 -0700 Subject: [PATCH 07/29] feat: support Duo hinge control and secondary display capture --- README.md | 55 +++++++++++--- .../Categories/XCUIDevice+FBHelpers.h | 10 +++ .../Categories/XCUIDevice+FBHelpers.m | 71 +++++++++++++++++++ WebDriverAgentLib/Commands/FBCustomCommands.m | 21 ++++++ .../Routing/FBScreenRecordingRequest.m | 9 ++- .../Utilities/FBActiveAppDetectionPoint.h | 4 +- .../Utilities/FBActiveAppDetectionPoint.m | 20 ++++++ WebDriverAgentLib/Utilities/FBConfiguration.h | 2 +- WebDriverAgentLib/Utilities/FBMjpegServer.m | 6 +- .../IntegrationTests/FBScreenTests.m | 49 +++++++++++++ .../UnitTests/FBCustomCommandsTests.m | 32 +++++++++ 11 files changed, 263 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 345e1d1571..1b42f28852 100644 --- a/README.md +++ b/README.md @@ -10,12 +10,13 @@ WebDriverAgent is a [WebDriver server](https://w3c.github.io/webdriver/webdriver-spec.html) implementation for iOS that can be used to remote control iOS devices. It allows you to launch & kill applications, tap & scroll views or confirm view presence on a screen. This makes it a perfect tool for application end-to-end testing or general purpose device automation. It works by linking `XCTest.framework` and calling Apple's API to execute commands directly on a device. WebDriverAgent is developed for end-to-end testing and is successfully adopted by [Appium](http://appium.io) via [XCUITest driver](https://github.com/appium/appium-xcuitest-driver). ## Features - * Both iOS and tvOS platforms are supported with devices & simulators - * Implements most of [WebDriver Spec](https://w3c.github.io/webdriver/webdriver-spec.html) - * Implements part of [Mobile JSON Wire Protocol Spec](https://github.com/SeleniumHQ/mobile-spec/blob/master/spec-draft.md) - * USB support for devices is implemented via [appium-ios-device](https://github.com/appium/appium-ios-device) library and has zero dependencies on third-party tools. - * Easy development cycle as it can be launched & debugged directly via Xcode - * Use [Mac2Driver](https://github.com/appium/appium-mac2-driver) to automate macOS apps + +- Both iOS and tvOS platforms are supported with devices & simulators +- Implements most of [WebDriver Spec](https://w3c.github.io/webdriver/webdriver-spec.html) +- Implements part of [Mobile JSON Wire Protocol Spec](https://github.com/SeleniumHQ/mobile-spec/blob/master/spec-draft.md) +- USB support for devices is implemented via [appium-ios-device](https://github.com/appium/appium-ios-device) library and has zero dependencies on third-party tools. +- Easy development cycle as it can be launched & debugged directly via Xcode +- Use [Mac2Driver](https://github.com/appium/appium-mac2-driver) to automate macOS apps ## Getting Started On This Repository @@ -26,21 +27,59 @@ and start sending [requests](https://github.com/facebook/WebDriverAgent/wiki/Que More about how to start WebDriverAgent [here](https://github.com/facebook/WebDriverAgent/wiki/Starting-WebDriverAgent). +## Multiple displays + +On devices such as iPhone Duo, `GET /wda/screens` lists the available displays. +Set `currentDisplayId` through `POST /session/:sessionId/appium/settings` to select +one of these IDs. Do not assume that the main display is the currently visible +one after folding or unfolding the device. + +The setting selects the display for screenshots, MJPEG frames, and newly started +XCTest screen recordings. MJPEG follows setting changes on the existing +connection; an in-progress recording keeps the display selected when it started. +An unavailable display causes screenshots and recording requests to fail; MJPEG +pauses frame delivery until a valid display is selected. Set `currentDisplayId` +to `null` to restore the main display. WDA does not automatically switch this +setting when the device changes pose. + +XCTest's active-app point lookup only targets the main display. When a secondary +display is selected, WDA uses its active-app fallback instead of that point lookup. +This is not display-specific app filtering: set `defaultActiveApplication` to the +app's bundle ID when multiple foreground apps make the result ambiguous. An app +specified when creating the session retains its existing priority. + +W3C coordinate actions require the separate display-aware actions support tracked +in [#1269](https://github.com/appium/WebDriverAgent/pull/1269). + +### Simulated hinge angle + +On an iPhone Duo simulator, `POST /session/:sessionId/wda/device/hingeAngle` with +`{"angle": 90}` sends a hinge event without operating Device Hub's UI. The angle +must be a finite number from `0` (closed) to `180` (fully open); fractional values +are supported. This uses the simulator's private HID protocol verified with +Xcode 27.1 and is currently limited to model `iPhone19,4`. Other simulators and +physical devices return `unsupported operation`. + +The response confirms event dispatch. Wait for the expected app layout before +continuing, since folding completes asynchronously. The command does not rotate +the device or update `currentDisplayId`; select the desired display separately. + ## Known Issues + If you are having some issues please checkout [wiki](https://github.com/facebook/WebDriverAgent/wiki/Common-Issues) first. ## For Contributors + If you want to help us out, you are more than welcome to. However please make sure you have followed the guidelines in [CONTRIBUTING](CONTRIBUTING.md). ## Creating Bundles `npm run bundle` -Then, you find `WebDriverAgentRunner-Runner-sim-.zip` for iOS and `WebDriverAgentRunner-Runner-tv_sim-.zip` for tvOS files in the current directory. +Then, you find `WebDriverAgentRunner-Runner-sim-.zip` for iOS and `WebDriverAgentRunner-Runner-tv_sim-.zip` for tvOS files in the current directory. ## License [`WebDriverAgent` is BSD-licensed](LICENSE). - Have fun! diff --git a/WebDriverAgentLib/Categories/XCUIDevice+FBHelpers.h b/WebDriverAgentLib/Categories/XCUIDevice+FBHelpers.h index 761571cdea..593717c983 100644 --- a/WebDriverAgentLib/Categories/XCUIDevice+FBHelpers.h +++ b/WebDriverAgentLib/Categories/XCUIDevice+FBHelpers.h @@ -231,6 +231,16 @@ typedef NS_ENUM(NSUInteger, FBUIInterfaceAppearance) { - (BOOL)fb_clearSimulatedLocation:(NSError **)error; #endif +/** Whether the current runtime is the supported iPhone Duo simulator. */ +- (BOOL)fb_supportsSimulatedHingeAngle; + +/** + Sends a simulated hinge angle in degrees (0 = closed, 180 = fully open). + This dispatches the event; the display transition completes asynchronously. + Only supported on the iPhone Duo simulator. Does not change currentDisplayId. + */ +- (BOOL)fb_setSimulatedHingeAngle:(double)angle error:(NSError **)error; + @end NS_ASSUME_NONNULL_END diff --git a/WebDriverAgentLib/Categories/XCUIDevice+FBHelpers.m b/WebDriverAgentLib/Categories/XCUIDevice+FBHelpers.m index 65b9785c39..c9aa2ae8eb 100644 --- a/WebDriverAgentLib/Categories/XCUIDevice+FBHelpers.m +++ b/WebDriverAgentLib/Categories/XCUIDevice+FBHelpers.m @@ -12,6 +12,8 @@ #import #include #import +#import +#import #import "FBErrorBuilder.h" #import "FBImageUtils.h" @@ -475,4 +477,73 @@ - (BOOL)fb_clearSimulatedLocation:(NSError **)error } #endif +- (BOOL)fb_supportsSimulatedHingeAngle +{ +#if TARGET_OS_SIMULATOR && !TARGET_OS_TV && !TARGET_OS_WATCH + // This vendor event is specific to Duo's simulated hardware. Do not send it + // to physical devices or assume every device with multiple screens supports it. + return [NSProcessInfo.processInfo.environment[@"SIMULATOR_MODEL_IDENTIFIER"] isEqualToString:@"iPhone19,4"]; +#else + return NO; +#endif +} + +- (BOOL)fb_setSimulatedHingeAngle:(double)angle error:(NSError **)error +{ + if (!isfinite(angle) || angle < 0 || angle > 180) { + return [[FBErrorBuilder.builder withDescription:@"Hinge angle must be a finite number between 0 and 180 degrees"] buildError:error]; + } + if (!self.fb_supportsSimulatedHingeAngle) { + return [[FBErrorBuilder.builder withDescription:@"Simulated hinge angle is only supported on the iPhone Duo simulator"] buildError:error]; + } +#if TARGET_OS_SIMULATOR && !TARGET_OS_TV && !TARGET_OS_WATCH + static CFDataRef (*serialize)(CFTypeRef, CFOptionFlags); + static CFTypeRef (*createEvent)(CFAllocatorRef, uint64_t, uint32_t, uint32_t, uint32_t, uint8_t *, CFIndex, uint32_t); + static CFTypeRef (*createClient)(CFAllocatorRef); + static void (*dispatchEvent)(CFTypeRef, CFTypeRef); + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + void *handle = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_LAZY); + if (NULL != handle) { + serialize = (CFDataRef (*)(CFTypeRef, CFOptionFlags))dlsym(handle, "IOCFSerialize"); + createEvent = (CFTypeRef (*)(CFAllocatorRef, uint64_t, uint32_t, uint32_t, uint32_t, uint8_t *, CFIndex, uint32_t))dlsym(handle, "IOHIDEventCreateVendorDefinedEvent"); + createClient = (CFTypeRef (*)(CFAllocatorRef))dlsym(handle, "IOHIDEventSystemClientCreate"); + dispatchEvent = (void (*)(CFTypeRef, CFTypeRef))dlsym(handle, "IOHIDEventSystemClientDispatchEvent"); + } + }); + if (NULL == serialize || NULL == createEvent || NULL == createClient || NULL == dispatchEvent) { + return [[FBErrorBuilder.builder withDescription:@"The simulator runtime does not provide the required IOKit HID APIs"] buildError:error]; + } + // Matches Device Hub's hinge-slider-control payload (Xcode 27.1). It is an + // IOCF binary serialization, not an NSPropertyListSerialization binary plist. + NSDictionary *payload = @{ + @"provider": @"com.apple.Virtualization.VirtualMachines", + @"source": @"hinge-slider-control", + @"type": @"range", + @"value": @(angle), + }; + CFDataRef data = serialize((__bridge CFTypeRef)payload, 1); + if (NULL == data) { + return [[FBErrorBuilder.builder withDescription:@"Cannot serialize the simulated hinge event"] buildError:error]; + } + CFTypeRef event = createEvent(kCFAllocatorDefault, mach_absolute_time(), 0xff61, 0x5b, 0, + (uint8_t *)CFDataGetBytePtr(data), CFDataGetLength(data), 0); + CFRelease(data); + CFTypeRef client = createClient(kCFAllocatorDefault); + BOOL canDispatch = NULL != event && NULL != client; + if (canDispatch) { + dispatchEvent(client, event); + } + if (NULL != event) { + CFRelease(event); + } + if (NULL != client) { + CFRelease(client); + } + return canDispatch || [[FBErrorBuilder.builder withDescription:@"Cannot create the simulated hinge HID event or client"] buildError:error]; +#else + return NO; +#endif +} + @end diff --git a/WebDriverAgentLib/Commands/FBCustomCommands.m b/WebDriverAgentLib/Commands/FBCustomCommands.m index c981d1b33a..700fca0fde 100644 --- a/WebDriverAgentLib/Commands/FBCustomCommands.m +++ b/WebDriverAgentLib/Commands/FBCustomCommands.m @@ -74,6 +74,7 @@ + (NSArray *)routes [[FBRoute POST:@"/wda/expectNotification"] respondWithTarget:self action:@selector(handleExpectNotification:)], [[FBRoute POST:@"/wda/siri/activate"] respondWithTarget:self action:@selector(handleActivateSiri:)], [[FBRoute POST:@"/wda/apps/launchUnattached"].withoutSession respondWithTarget:self action:@selector(handleLaunchUnattachedApp:)], + [[FBRoute POST:@"/wda/device/hingeAngle"] respondWithTarget:self action:@selector(handleSetSimulatedHingeAngle:)], [[FBRoute GET:@"/wda/device/info"] respondWithTarget:self action:@selector(handleGetDeviceInfo:)], [[FBRoute POST:@"/wda/resetAppAuth"] respondWithTarget:self action:@selector(handleResetAppAuth:)], [[FBRoute GET:@"/wda/device/info"].withoutSession respondWithTarget:self action:@selector(handleGetDeviceInfo:)], @@ -777,4 +778,24 @@ + (NSString *)timeZone return FBResponseWithObject(result); } ++ (id)handleSetSimulatedHingeAngle:(FBRouteRequest *)request +{ + id angle = request.arguments[@"angle"]; + if (![angle isKindOfClass:NSNumber.class] + || CFGetTypeID((__bridge CFTypeRef)angle) == CFBooleanGetTypeID() + || !isfinite([angle doubleValue]) || [angle doubleValue] < 0 || [angle doubleValue] > 180) { + return FBResponseWithStatus([FBCommandStatus invalidArgumentErrorWithMessage: + @"'angle' must be a finite number between 0 and 180 degrees" traceback:nil]); + } + if (!XCUIDevice.sharedDevice.fb_supportsSimulatedHingeAngle) { + return FBResponseWithStatus([FBCommandStatus unsupportedOperationErrorWithMessage: + @"Simulated hinge angle is only supported on the iPhone Duo simulator" traceback:nil]); + } + NSError *error; + if (![XCUIDevice.sharedDevice fb_setSimulatedHingeAngle:[angle doubleValue] error:&error]) { + return FBResponseWithUnknownError(error); + } + return FBResponseWithOK(); +} + @end diff --git a/WebDriverAgentLib/Routing/FBScreenRecordingRequest.m b/WebDriverAgentLib/Routing/FBScreenRecordingRequest.m index 06f9d77906..d572df9481 100644 --- a/WebDriverAgentLib/Routing/FBScreenRecordingRequest.m +++ b/WebDriverAgentLib/Routing/FBScreenRecordingRequest.m @@ -9,6 +9,7 @@ #import "FBScreenRecordingRequest.h" #import "FBErrorBuilder.h" +#import "FBScreen.h" #import "XCUIScreen.h" @implementation FBScreenRecordingRequest @@ -56,6 +57,10 @@ - (nullable id)createVideoEncodingWithError:(NSError **)error - (id)toNativeRequestWithError:(NSError **)error { + XCUIScreen *screen = [FBScreen currentScreenWithError:error]; + if (nil == screen) { + return nil; + } Class screenRecordingRequestClass = NSClassFromString(@"XCTScreenRecordingRequest"); if (nil == screenRecordingRequestClass) { [[[FBErrorBuilder builder] @@ -80,8 +85,8 @@ - (id)toNativeRequestWithError:(NSError **)error NSMethodSignature *screenRecordingRequestContructorSignature = [screenRecordingRequestAllocated methodSignatureForSelector:screenRecordingRequestConstructorSelector]; NSInvocation *screenRecordingRequestInitInvocation = [NSInvocation invocationWithMethodSignature:screenRecordingRequestContructorSignature]; [screenRecordingRequestInitInvocation setSelector:screenRecordingRequestConstructorSelector]; - long long mainScreenId = XCUIScreen.mainScreen.displayID; - [screenRecordingRequestInitInvocation setArgument:&mainScreenId atIndex:2]; + long long screenId = screen.displayID; + [screenRecordingRequestInitInvocation setArgument:&screenId atIndex:2]; CGRect fullScreenRect = CGRectNull; [screenRecordingRequestInitInvocation setArgument:&fullScreenRect atIndex:3]; [screenRecordingRequestInitInvocation setArgument:&videoEncoding atIndex:4]; diff --git a/WebDriverAgentLib/Utilities/FBActiveAppDetectionPoint.h b/WebDriverAgentLib/Utilities/FBActiveAppDetectionPoint.h index a65bd38800..146af425d8 100644 --- a/WebDriverAgentLib/Utilities/FBActiveAppDetectionPoint.h +++ b/WebDriverAgentLib/Utilities/FBActiveAppDetectionPoint.h @@ -24,14 +24,14 @@ NS_ASSUME_NONNULL_BEGIN * Calculates the accessbility element which is located at the given screen coordinates * * @param point The screen coordinates - * @returns The retrieved accessbility element or nil if it cannot be detected + * @returns The retrieved accessbility element or nil if it cannot be detected or a secondary display is selected */ + (nullable id)axElementWithPoint:(CGPoint)point; /** * Retrieves the accessbility element for the current screen point * - * @returns The retrieved accessbility element or nil if it cannot be detected + * @returns The retrieved accessbility element or nil if it cannot be detected or a secondary display is selected */ - (nullable id)axElement; diff --git a/WebDriverAgentLib/Utilities/FBActiveAppDetectionPoint.m b/WebDriverAgentLib/Utilities/FBActiveAppDetectionPoint.m index c24a8fbd72..534f5020af 100644 --- a/WebDriverAgentLib/Utilities/FBActiveAppDetectionPoint.m +++ b/WebDriverAgentLib/Utilities/FBActiveAppDetectionPoint.m @@ -13,6 +13,10 @@ #endif #import "FBErrorBuilder.h" +#import "FBScreen.h" +#import "FBConfiguration.h" +#import "FBExceptions.h" +#import "XCUIScreen.h" #import "FBLogger.h" #import "FBXCTestDaemonsProxy.h" #import "XCTMessagingChannel_RunnerToDaemon-Protocol.h" @@ -45,6 +49,22 @@ + (instancetype)sharedInstance + (id)axElementWithPoint:(CGPoint)point { + if (nil != FBConfiguration.sharedInstance.currentDisplayId) { + NSError *error; + XCUIScreen *screen = [FBScreen currentScreenWithError:&error]; + if (nil == screen) { + @throw [NSException exceptionWithName:FBInvalidArgumentException + reason:error.localizedDescription + userInfo:nil]; + } + // XCTest's point lookup has no display parameter and always hits the main + // display. On Duo this can select the outer SpringBoard while the app is + // foreground on the inner display. Let callers use their active-app fallback + // instead of treating an element from a different display as authoritative. + if (!screen.isMainScreen) { + return nil; + } + } __block id onScreenElement = nil; id proxy = [FBXCTestDaemonsProxy testRunnerProxy]; dispatch_semaphore_t sem = dispatch_semaphore_create(0); diff --git a/WebDriverAgentLib/Utilities/FBConfiguration.h b/WebDriverAgentLib/Utilities/FBConfiguration.h index d905c2be1e..763faa1164 100644 --- a/WebDriverAgentLib/Utilities/FBConfiguration.h +++ b/WebDriverAgentLib/Utilities/FBConfiguration.h @@ -369,7 +369,7 @@ typedef NS_ENUM(NSInteger, FBConfigurationKeyboardPreference) { @property (atomic, assign) BOOL enforceCustomSnapshots; /** - * The identifier of the display targeted by screenshots. + * The identifier of the display targeted by screenshots, MJPEG, and new screen recordings. * nil means the main display. Reset to nil for each new session. */ @property (atomic, copy, nullable) NSNumber *currentDisplayId; diff --git a/WebDriverAgentLib/Utilities/FBMjpegServer.m b/WebDriverAgentLib/Utilities/FBMjpegServer.m index 2e6560eb7f..0b4a67f25c 100644 --- a/WebDriverAgentLib/Utilities/FBMjpegServer.m +++ b/WebDriverAgentLib/Utilities/FBMjpegServer.m @@ -16,6 +16,7 @@ #import "FBConfiguration.h" #import "FBLogger.h" #import "FBScreenshot.h" +#import "FBScreen.h" #import "FBImageProcessor.h" #import "FBImageUtils.h" #import "XCUIScreen.h" @@ -42,7 +43,6 @@ @interface FBMjpegServer() @property (nonatomic, readonly) dispatch_queue_t backgroundQueue; @property (nonatomic, readonly) NSMutableArray *listeningClients; @property (nonatomic, readonly) FBImageProcessor *imageProcessor; -@property (nonatomic, readonly) long long mainScreenID; @property (nonatomic, assign) NSUInteger consecutiveScreenshotFailures; @property (atomic, assign) BOOL isStreaming; @property (nonatomic, assign) NSUInteger sentFramesCount; @@ -67,7 +67,6 @@ - (instancetype)init _pendingFrameCounts = [NSMapTable mapTableWithKeyOptions:(NSPointerFunctionsOptions)(NSMapTableObjectPointerPersonality | NSMapTableStrongMemory) valueOptions:(NSPointerFunctionsOptions)NSMapTableStrongMemory]; _imageProcessor = [[FBImageProcessor alloc] init]; - _mainScreenID = [XCUIScreen.mainScreen displayID]; dispatch_queue_attr_t queueAttributes = dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL, QOS_CLASS_UTILITY, 0); _backgroundQueue = dispatch_queue_create(QUEUE_NAME, queueAttributes); __weak typeof(self) weakSelf = self; @@ -116,7 +115,8 @@ - (void)streamScreenshot NSError *error; CGFloat compressionQuality = MAX(FBMinCompressionQuality, MIN(FBMaxCompressionQuality, (double)FBConfiguration.sharedInstance.mjpegServerScreenshotQuality / 100.0)); - NSData *screenshotData = [FBScreenshot takeInOriginalResolutionWithScreenID:self.mainScreenID + XCUIScreen *screen = [FBScreen currentScreenWithError:&error]; + NSData *screenshotData = nil == screen ? nil : [FBScreenshot takeInOriginalResolutionWithScreenID:screen.displayID compressionQuality:compressionQuality uti:UTTypeJPEG timeout:FRAME_TIMEOUT diff --git a/WebDriverAgentTests/IntegrationTests/FBScreenTests.m b/WebDriverAgentTests/IntegrationTests/FBScreenTests.m index 012b2614b5..e2cd6cc48c 100644 --- a/WebDriverAgentTests/IntegrationTests/FBScreenTests.m +++ b/WebDriverAgentTests/IntegrationTests/FBScreenTests.m @@ -11,6 +11,9 @@ #import "FBIntegrationTestCase.h" #import "FBConfiguration.h" #import "FBScreen.h" +#import "FBActiveAppDetectionPoint.h" +#import "FBExceptions.h" +#import "FBScreenRecordingRequest.h" #import "FBScreenshot.h" #import "XCUIScreen.h" @@ -108,6 +111,52 @@ - (void)testSessionResetClearsCurrentDisplay XCTAssertNil(FBConfiguration.sharedInstance.currentDisplayId); } +- (void)testSecondaryDisplayDoesNotHitTestMainDisplay +{ + NSArray *> *screens = [FBScreen screensWithError:nil]; + BOOL testedSecondaryDisplay = NO; + for (NSDictionary *screen in screens) { + if ([screen[@"isMain"] boolValue]) { + continue; + } + testedSecondaryDisplay = YES; + FBConfiguration.sharedInstance.currentDisplayId = screen[@"displayId"]; + XCTAssertNil(FBActiveAppDetectionPoint.sharedInstance.axElement); + } + XCTSkipIf(!testedSecondaryDisplay, @"Requires a secondary display"); +} + +- (void)testActiveAppDetectionRejectsUnavailableDisplay +{ + FBConfiguration.sharedInstance.currentDisplayId = @([self unknownDisplayID]); + XCTAssertThrowsSpecificNamed(FBActiveAppDetectionPoint.sharedInstance.axElement, + NSException, FBInvalidArgumentException); +} + +- (void)testRecordingUsesSelectedDisplay +{ + XCTSkipIf(nil == NSClassFromString(@"XCTScreenRecordingRequest"), @"Screen recording is unavailable"); + FBScreenRecordingRequest *request = [[FBScreenRecordingRequest alloc] initWithFps:10 codec:0]; + NSError *error = nil; + id defaultRequest = [request toNativeRequestWithError:&error]; + XCTAssertNotNil(defaultRequest); + XCTAssertNil(error); + XCTAssertEqualObjects([defaultRequest valueForKey:@"screenID"], @([FBScreen displayID])); + + for (NSDictionary *screen in [FBScreen screensWithError:nil]) { + FBConfiguration.sharedInstance.currentDisplayId = screen[@"displayId"]; + id nativeRequest = [request toNativeRequestWithError:&error]; + XCTAssertNotNil(nativeRequest); + XCTAssertNil(error); + XCTAssertEqualObjects([nativeRequest valueForKey:@"screenID"], screen[@"displayId"]); + } + + FBConfiguration.sharedInstance.currentDisplayId = @([self unknownDisplayID]); + XCTAssertNil([request toNativeRequestWithError:&error]); + XCTAssertNotNil(error); + XCTAssertTrue([error.localizedDescription containsString:@"Available display ids"]); +} + - (long long)unknownDisplayID { long long maxID = 0; diff --git a/WebDriverAgentTests/UnitTests/FBCustomCommandsTests.m b/WebDriverAgentTests/UnitTests/FBCustomCommandsTests.m index e8c106a6dc..842c07ceb5 100644 --- a/WebDriverAgentTests/UnitTests/FBCustomCommandsTests.m +++ b/WebDriverAgentTests/UnitTests/FBCustomCommandsTests.m @@ -12,12 +12,16 @@ #import "FBElementCache.h" #import "FBRouteRequest-Private.h" #import "FBSession.h" +#import "FBResponsePayload.h" +#import "RouteResponse.h" +#import "XCUIDevice+FBHelpers.h" #import "Doubles/XCUIElementDouble.h" #if !TARGET_OS_TV && __clang_major__ >= 15 @interface FBCustomCommands (FBWDATestable) + (id)handleKeyboardInput:(FBRouteRequest *)request; ++ (id)handleSetSimulatedHingeAngle:(FBRouteRequest *)request; @end @interface FBCustomCommandsTests : XCTestCase @@ -78,6 +82,34 @@ - (void)testDictionaryKeyWithConstantNameAndModifierFlagsIsResolved XCTAssertEqual(element.lastTypedModifierFlags, 2); } +- (NSDictionary *)hingeResponseWithArguments:(NSDictionary *)arguments +{ + FBRouteRequest *request = [FBRouteRequest routeRequestWithURL:[NSURL URLWithString:@"http://localhost:8100/"] + parameters:@{} + arguments:arguments]; + RouteResponse *response = [RouteResponse new]; + [[FBCustomCommands handleSetSimulatedHingeAngle:request] dispatchWithResponse:response]; + return [NSJSONSerialization JSONObjectWithData:response.responseData options:0 error:nil]; +} + +- (void)testHingeAngleRejectsInvalidArguments +{ + for (id value in @[@(-1), @181, @YES, @"90", NSNull.null, @(NAN), @(INFINITY)]) { + NSDictionary *response = [self hingeResponseWithArguments:@{@"angle": value}]; + XCTAssertEqualObjects(response[@"value"][@"error"], @"invalid argument"); + } + XCTAssertEqualObjects([self hingeResponseWithArguments:@{}][@"value"][@"error"], @"invalid argument"); +} + +- (void)testHingeAngleRejectsUnsupportedDevices +{ + XCTSkipIf(XCUIDevice.sharedDevice.fb_supportsSimulatedHingeAngle, @"Requires a device without a simulated hinge"); + for (NSNumber *angle in @[@0, @90, @180]) { + XCTAssertEqualObjects([self hingeResponseWithArguments:@{@"angle": angle}][@"value"][@"error"], + @"unsupported operation"); + } +} + @end #endif From 16a38f3ea2b81534910f90ceccad0bb1f15c3ee7 Mon Sep 17 00:00:00 2001 From: Kazuaki Matsuo Date: Sat, 3 Oct 2026 00:52:29 -0700 Subject: [PATCH 08/29] fix: align element actions and scrolling on secondary displays --- README.md | 8 +- .../Categories/XCUIElement+FBScrolling.m | 35 +++- WebDriverAgentLib/Utilities/FBMathUtils.h | 3 + WebDriverAgentLib/Utilities/FBMathUtils.m | 21 +++ .../Utilities/FBW3CActionsSynthesizer.m | 47 +++-- .../IntegrationTests/FBScreenTests.m | 169 ++++++++++++++++++ .../UnitTests/FBMathUtilsTests.m | 15 ++ 7 files changed, 272 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 1b42f28852..5c1ea5e650 100644 --- a/README.md +++ b/README.md @@ -48,8 +48,12 @@ This is not display-specific app filtering: set `defaultActiveApplication` to th app's bundle ID when multiple foreground apps make the result ambiguous. An app specified when creating the session retains its existing priority. -W3C coordinate actions require the separate display-aware actions support tracked -in [#1269](https://github.com/appium/WebDriverAgent/pull/1269). +W3C touch actions target the selected display. Viewport, element, and pointer +origins can be mixed within one action sequence; element offsets remain relative +to the element's center. Element scrolling uses the visible area in the same +coordinate space as the element frame, including on a rotated secondary display. + +This gesture support builds on [#1269](https://github.com/appium/WebDriverAgent/pull/1269). ### Simulated hinge angle diff --git a/WebDriverAgentLib/Categories/XCUIElement+FBScrolling.m b/WebDriverAgentLib/Categories/XCUIElement+FBScrolling.m index 85a2386758..8a90b1f133 100644 --- a/WebDriverAgentLib/Categories/XCUIElement+FBScrolling.m +++ b/WebDriverAgentLib/Categories/XCUIElement+FBScrolling.m @@ -10,6 +10,7 @@ #import "FBErrorBuilder.h" #import "FBLogger.h" +#import "FBScreen.h" #import "FBMacros.h" #import "FBMathUtils.h" #import "FBXCodeCompatibility.h" @@ -20,6 +21,7 @@ #import "XCUIElement+FBUID.h" #import "XCUIApplication.h" #import "XCUICoordinate.h" +#import "XCUIScreen.h" #import "XCUIElement+FBIsVisible.h" #import "XCUIElement+FBVisibleFrame.h" #import "XCUIElement.h" @@ -35,6 +37,20 @@ #if !TARGET_OS_TV +static CGRect FBScrollFrameInApplication(CGRect frame, XCUIApplication *application) +{ + XCUIScreen *screen = [FBScreen currentScreenWithError:nil]; + if (nil != screen && !screen.isMainScreen) { + // On secondary displays, XCTest's visibleFrame is in portrait screen + // coordinates, whereas wdFrame uses the application's current orientation. + // Rotate the clipped visible rect before normalizing it against the anchor. + CGSize size = CGSizeMake(CGRectGetWidth(screen.bounds) / screen.scale, + CGRectGetHeight(screen.bounds) / screen.scale); + frame = FBRectFromPortraitCoordinates(frame, size, application.interfaceOrientation); + } + return frame; +} + @interface FBXCElementSnapshotWrapper (FBScrolling) - (BOOL)fb_scrollUpByNormalizedDistance:(CGFloat)distance anchorElement:(XCUIElement *)anchorElement; @@ -260,7 +276,8 @@ - (BOOL)fb_scrollToVisibleWithNormalizedScrollDistance:(CGFloat)normalizedScroll // Trying fb_cachedSnapshot first FBXCElementSnapshotWrapper *targetCellSnapshotWrapped = [FBXCElementSnapshotWrapper ensureWrapped:[self fb_customSnapshot]]; targetCellSnapshot = [targetCellSnapshotWrapped fb_parentCellSnapshot]; - CGRect visibleFrame = [FBXCElementSnapshotWrapper ensureWrapped:targetCellSnapshot].fb_visibleFrame; + CGRect visibleFrame = FBScrollFrameInApplication( + [FBXCElementSnapshotWrapper ensureWrapped:targetCellSnapshot].fb_visibleFrame, self.application); CGVector scrollVector = CGVectorMake(visibleFrame.size.width - targetCellSnapshot.frame.size.width, visibleFrame.size.height - targetCellSnapshot.frame.size.height @@ -296,9 +313,9 @@ - (BOOL)fb_isEquivalentElementSnapshotVisible:(id)snapshot @implementation FBXCElementSnapshotWrapper (FBScrolling) -- (CGRect)scrollingFrame +- (CGRect)scrollingFrameWithAnchor:(XCUIElement *)anchorElement { - return self.visibleFrame; + return FBScrollFrameInApplication(self.visibleFrame, anchorElement.application); } - (BOOL)fb_scrollUpByNormalizedDistance:(CGFloat)distance @@ -328,8 +345,9 @@ - (BOOL)fb_scrollRightByNormalizedDistance:(CGFloat)distance - (BOOL)fb_scrollByNormalizedVector:(CGVector)normalizedScrollVector anchorElement:(XCUIElement *)anchorElement { - CGVector scrollVector = CGVectorMake(CGRectGetWidth(self.scrollingFrame) * normalizedScrollVector.dx, - CGRectGetHeight(self.scrollingFrame) * normalizedScrollVector.dy + CGRect frame = [self scrollingFrameWithAnchor:anchorElement]; + CGVector scrollVector = CGVectorMake(CGRectGetWidth(frame) * normalizedScrollVector.dx, + CGRectGetHeight(frame) * normalizedScrollVector.dy ); return [self fb_scrollByVector:scrollVector anchorElement:anchorElement error:nil]; } @@ -338,9 +356,10 @@ - (BOOL)fb_scrollByVector:(CGVector)vector anchorElement:(XCUIElement *)anchorElement error:(NSError **)error { + CGRect frame = [self scrollingFrameWithAnchor:anchorElement]; CGVector scrollBoundingVector = CGVectorMake( - CGRectGetWidth(self.scrollingFrame) * FBScrollTouchProportion, - CGRectGetHeight(self.scrollingFrame) * FBScrollTouchProportion + CGRectGetWidth(frame) * FBScrollTouchProportion, + CGRectGetHeight(frame) * FBScrollTouchProportion ); scrollBoundingVector.dx = (CGFloat)floor(copysign(scrollBoundingVector.dx, vector.dx)); scrollBoundingVector.dy = (CGFloat)floor(copysign(scrollBoundingVector.dy, vector.dy)); @@ -373,7 +392,7 @@ - (BOOL)fb_scrollAncestorScrollViewByVectorWithinScrollViewFrame:(CGVector)vecto anchorElement:(XCUIElement *)anchorElement error:(NSError **)error { - CGRect scrollingFrame = self.scrollingFrame; + CGRect scrollingFrame = [self scrollingFrameWithAnchor:anchorElement]; // wdFrame matches scrollingFrame's coordinate space; raw .frame can be pre-scaled or // dimension-swapped and drift out of sync with it (appium/appium#16185). CGRect anchorFrame = anchorElement.wdFrame; diff --git a/WebDriverAgentLib/Utilities/FBMathUtils.h b/WebDriverAgentLib/Utilities/FBMathUtils.h index ebce725ebc..cbd5f8144c 100644 --- a/WebDriverAgentLib/Utilities/FBMathUtils.h +++ b/WebDriverAgentLib/Utilities/FBMathUtils.h @@ -35,6 +35,9 @@ BOOL FBSizeFuzzyEqualToSize(CGSize size1, CGSize size2, CGFloat threshold); BOOL FBRectFuzzyEqualToRect(CGRect rect1, CGRect rect2, CGFloat threshold); #if !TARGET_OS_TV && !TARGET_OS_WATCH +/*! Converts a portrait-screen rect into the application's orientation, preserving its clipping. */ +CGRect FBRectFromPortraitCoordinates(CGRect frame, CGSize screenSize, UIInterfaceOrientation orientation); + /*! Inverts size if necessary to match current screen orientation */ CGSize FBAdjustDimensionsForApplication(CGSize actualSize, UIInterfaceOrientation orientation); diff --git a/WebDriverAgentLib/Utilities/FBMathUtils.m b/WebDriverAgentLib/Utilities/FBMathUtils.m index 0e1faa477f..8d12b1d20e 100644 --- a/WebDriverAgentLib/Utilities/FBMathUtils.m +++ b/WebDriverAgentLib/Utilities/FBMathUtils.m @@ -50,6 +50,27 @@ BOOL FBRectFuzzyEqualToRect(CGRect rect1, CGRect rect2, CGFloat threshold) #if !TARGET_OS_TV && !TARGET_OS_WATCH +CGRect FBRectFromPortraitCoordinates(CGRect frame, CGSize screenSize, UIInterfaceOrientation orientation) +{ + if (CGRectIsEmpty(frame)) { + return frame; + } + CGFloat width = MIN(screenSize.width, screenSize.height); + CGFloat height = MAX(screenSize.width, screenSize.height); + switch (orientation) { + case UIInterfaceOrientationLandscapeLeft: + return CGRectMake(height - CGRectGetMaxY(frame), CGRectGetMinX(frame), frame.size.height, frame.size.width); + case UIInterfaceOrientationLandscapeRight: + return CGRectMake(CGRectGetMinY(frame), width - CGRectGetMaxX(frame), frame.size.height, frame.size.width); + case UIInterfaceOrientationPortraitUpsideDown: + return CGRectMake(width - CGRectGetMaxX(frame), height - CGRectGetMaxY(frame), frame.size.width, frame.size.height); + case UIInterfaceOrientationUnknown: + case UIInterfaceOrientationPortrait: + default: + return frame; + } +} + CGSize FBAdjustDimensionsForApplication(CGSize actualSize, UIInterfaceOrientation orientation) { if (orientation == UIInterfaceOrientationLandscapeLeft || orientation == UIInterfaceOrientationLandscapeRight) { diff --git a/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m b/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m index 7c3a4f3f5a..2ff1214836 100644 --- a/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m +++ b/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m @@ -68,7 +68,10 @@ #if !TARGET_OS_TV && !TARGET_OS_WATCH @interface FBW3CGestureItem : FBBaseGestureItem -@property (nullable, readonly, nonatomic) FBBaseGestureItem *previousItem; +@property (nullable, readonly, nonatomic) FBW3CGestureItem *previousItem; +@property (nonatomic) CGVector displayCorrection; +@property (nonatomic) BOOL usesDisplayCorrection; +@property (readonly, nonatomic) CGPoint screenPoint; @end @@ -119,7 +122,7 @@ @implementation FBW3CGestureItem - (nullable instancetype)initWithActionItem:(NSDictionary *)actionItem application:(XCUIApplication *)application - previousItem:(nullable FBBaseGestureItem *)previousItem + previousItem:(nullable FBW3CGestureItem *)previousItem offset:(double)offset error:(NSError **)error { @@ -129,6 +132,7 @@ - (nullable instancetype)initWithActionItem:(NSDictionary *)acti self.application = application; self.offset = offset; _previousItem = previousItem; + _usesDisplayCorrection = previousItem.usesDisplayCorrection; NSNumber *durationObj = FBOptDuration(actionItem, @0, error); if (nil == durationObj) { return nil; @@ -144,6 +148,14 @@ - (nullable instancetype)initWithActionItem:(NSDictionary *)acti return self; } +- (CGPoint)screenPoint +{ + CGPoint point = self.atPosition.screenPoint; + return self.usesDisplayCorrection + ? CGPointMake(point.x + self.displayCorrection.dx, point.y + self.displayCorrection.dy) + : point; +} + - (nullable XCUICoordinate *)positionWithError:(NSError **)error { if (nil == self.previousItem || nil == self.previousItem.atPosition) { @@ -225,7 +237,7 @@ + (NSString *)actionName } } if (nil == self.pressure) { - XCPointerEventPath *result = [[XCPointerEventPath alloc] initForTouchAtPoint:self.atPosition.screenPoint + XCPointerEventPath *result = [[XCPointerEventPath alloc] initForTouchAtPoint:self.screenPoint offset:FBMillisToSeconds(self.offset)]; return @[result]; } @@ -282,6 +294,8 @@ - (nullable XCUICoordinate *)positionWithError:(NSError **)error } if (nil != element) { + // Normalized element coordinates already resolve against their own display. + self.usesDisplayCorrection = NO; if (nil == x && nil == y) { return [self hitpointWithElement:element positionOffset:nil error:error]; } @@ -289,6 +303,7 @@ - (nullable XCUICoordinate *)positionWithError:(NSError **)error } if ([origin isKindOfClass:NSString.class] && [origin isEqualToString:FB_ORIGIN_TYPE_VIEWPORT]) { + self.usesDisplayCorrection = YES; return [self hitpointWithElement:nil positionOffset:[NSValue valueWithCGPoint:CGPointMake(x.floatValue, y.floatValue)] error:error]; } @@ -302,6 +317,7 @@ - (nullable XCUICoordinate *)positionWithError:(NSError **)error } XCUICoordinate *recentPosition = self.previousItem.atPosition; CGVector offsetRelativeToRecentPosition = (nil == x && nil == y) ? CGVectorMake(0, 0) : CGVectorMake(x.floatValue, y.floatValue); + // Pointer-relative moves keep the coordinate space of their preceding position. return [recentPosition coordinateWithOffset:offsetRelativeToRecentPosition]; } @@ -316,10 +332,10 @@ + (NSString *)actionName error:(NSError **)error { if (nil == eventPath) { - return @[[[XCPointerEventPath alloc] initForTouchAtPoint:self.atPosition.screenPoint + return @[[[XCPointerEventPath alloc] initForTouchAtPoint:self.screenPoint offset:FBMillisToSeconds(self.offset + self.duration)]]; } - [eventPath moveToPoint:self.atPosition.screenPoint + [eventPath moveToPoint:self.screenPoint atOffset:FBMillisToSeconds(self.offset + self.duration)]; return @[]; } @@ -639,6 +655,10 @@ - (void)addItem:(FBBaseActionItem *)item @end +@interface FBW3CActionsSynthesizer () +@property (nonatomic) CGVector displayCorrection; +@end + @implementation FBW3CActionsSynthesizer - (NSArray *> *)preprocessedActionItemsWith:(NSArray *> *)actionItems @@ -823,6 +843,7 @@ @implementation FBW3CActionsSynthesizer return nil; } + gestureItem.displayCorrection = self.displayCorrection; [chain addItem:gestureItem]; } @@ -856,7 +877,7 @@ - (nullable XCSynthesizedEventRecord *)synthesizeWithError:(NSError **)error return nil; } XCSynthesizedEventRecord *eventRecord; - CGVector displayCorrection = CGVectorMake(0, 0); + self.displayCorrection = CGVectorMake(0, 0); if (screen.isMainScreen) { eventRecord = [[XCSynthesizedEventRecord alloc] initWithName:@"W3C Touch Action" interfaceOrientation:self.application.interfaceOrientation]; @@ -871,11 +892,11 @@ - (nullable XCSynthesizedEventRecord *)synthesizeWithError:(NSError **)error eventRecord = [[XCSynthesizedEventRecord alloc] initWithName:@"W3C Touch Action" displayID:(unsigned long long)screen.displayID interfaceOrientation:self.application.interfaceOrientation]; - // XCUICoordinate.screenPoint uses the main display's size when rotating - // coordinates. Selecting an event display does not change that transform; - // compensate for the difference before dispatching to the selected display. + // Viewport coordinates use the main display's size when rotating screenPoint. + // Element coordinates already use their own display. Apply this correction + // per gesture item so mixed origins and pointer-relative moves stay aligned. XCUIScreen *mainScreen = XCUIScreen.mainScreen; - displayCorrection = FBDisplayCoordinateOffset( + self.displayCorrection = FBDisplayCoordinateOffset( CGSizeMake(mainScreen.bounds.size.width / mainScreen.scale, mainScreen.bounds.size.height / mainScreen.scale), CGSizeMake(screen.bounds.size.width / screen.scale, screen.bounds.size.height / screen.scale), eventRecord.interfaceOrientation); @@ -921,12 +942,6 @@ - (nullable XCSynthesizedEventRecord *)synthesizeWithError:(NSError **)error return nil; } for (XCPointerEventPath *eventPath in eventPaths) { - if (displayCorrection.dx != 0 || displayCorrection.dy != 0) { - for (XCPointerEvent *pointerEvent in eventPath.pointerEvents) { - CGPoint point = pointerEvent.coordinate; - pointerEvent.coordinate = CGPointMake(point.x + displayCorrection.dx, point.y + displayCorrection.dy); - } - } [eventRecord addPointerEventPath:eventPath]; } } diff --git a/WebDriverAgentTests/IntegrationTests/FBScreenTests.m b/WebDriverAgentTests/IntegrationTests/FBScreenTests.m index e2cd6cc48c..1acb10a338 100644 --- a/WebDriverAgentTests/IntegrationTests/FBScreenTests.m +++ b/WebDriverAgentTests/IntegrationTests/FBScreenTests.m @@ -16,6 +16,8 @@ #import "FBScreenRecordingRequest.h" #import "FBScreenshot.h" #import "XCUIScreen.h" +#import "XCUIApplication+FBTouchAction.h" +#import "XCUIElement+FBScrolling.h" @interface FBScreenTests : FBIntegrationTestCase @end @@ -172,3 +174,170 @@ - (void)testScreenScale } @end + + +@interface FBDisplayGestureTests : FBIntegrationTestCase +@property (nonatomic) NSNumber *previousDisplayId; +@property (nonatomic) XCUIElement *canvas; +@end + +@implementation FBDisplayGestureTests + +- (void)setUp +{ + [super setUp]; + self.previousDisplayId = FBConfiguration.sharedInstance.currentDisplayId; + [self launchApplication]; + [self.testedApplication.buttons[@"coordinate-probe"] tap]; + self.canvas = self.testedApplication.otherElements[@"coordinate-canvas"]; + + // The fixture reports the screen it actually occupies. Do not assume the main + // screen is active: Duo changes displays when its hinge changes position. + NSArray *size = [self measurement][@"screenSize"]; + NSNumber *displayId = nil; + for (NSDictionary *screen in [FBScreen screensWithError:nil]) { + CGFloat width = [screen[@"bounds"][@"width"] doubleValue] / [screen[@"scale"] doubleValue]; + CGFloat height = [screen[@"bounds"][@"height"] doubleValue] / [screen[@"scale"] doubleValue]; + if (fabs(MAX(width, height) - MAX(size[0].doubleValue, size[1].doubleValue)) < 1 + && fabs(MIN(width, height) - MIN(size[0].doubleValue, size[1].doubleValue)) < 1) { + displayId = screen[@"displayId"]; + break; + } + } + XCTAssertNotNil(displayId, @"Cannot identify the fixture's display"); + FBConfiguration.sharedInstance.currentDisplayId = displayId; +} + +- (void)tearDown +{ + FBConfiguration.sharedInstance.currentDisplayId = self.previousDisplayId; + [super tearDown]; +} + +- (NSDictionary *)measurement +{ + NSString *value = self.testedApplication.staticTexts[@"probe-status"].label; + NSError *error = nil; + NSDictionary *result = [NSJSONSerialization JSONObjectWithData:[value dataUsingEncoding:NSUTF8StringEncoding] + options:0 error:&error]; + XCTAssertNil(error); + return result; +} + +- (void)performMoves:(NSArray *)moves expectedEnd:(CGPoint)expected +{ + NSUInteger count = [[self measurement][@"count"] unsignedIntegerValue]; + NSMutableArray *items = [NSMutableArray arrayWithArray:@[ + moves.firstObject, + @{@"type": @"pointerDown", @"button": @0}, + @{@"type": @"pause", @"duration": @100}, + ]]; + if (moves.count > 1) { + [items addObjectsFromArray:[moves subarrayWithRange:NSMakeRange(1, moves.count - 1)]]; + } + [items addObject:@{@"type": @"pointerUp", @"button": @0}]; + NSError *error = nil; + NSArray *actions = @[@{ + @"type": @"pointer", @"id": @"finger", @"parameters": @{@"pointerType": @"touch"}, @"actions": items, + }]; + XCTAssertTrue([self.testedApplication fb_performW3CActions:actions elementCache:nil error:&error], @"%@", error); + NSDictionary *result = [self measurement]; + XCTAssertEqual([result[@"count"] unsignedIntegerValue], count + 1); + XCTAssertEqualObjects(result[@"phase"], @"ended"); + NSArray *actual = result[@"last"]; + XCTAssertEqualWithAccuracy(actual[0].doubleValue, expected.x, 1); + XCTAssertEqualWithAccuracy(actual[1].doubleValue, expected.y, 1); +} + +- (NSDictionary *)moveFrom:(id)origin x:(CGFloat)x y:(CGFloat)y duration:(NSUInteger)duration +{ + return @{@"type": @"pointerMove", @"origin": origin, @"x": @(x), @"y": @(y), @"duration": @(duration)}; +} + +- (void)testViewportAndElementCorners +{ + NSDictionary *geometry = [self measurement]; + NSArray *bounds = geometry[@"canvasBounds"]; + NSArray *frame = geometry[@"canvasWindowRect"]; + CGFloat width = bounds[0].doubleValue, height = bounds[1].doubleValue; + for (NSNumber *x in @[@40, @(width - 40)]) { + for (NSNumber *y in @[@40, @(height - 40)]) { + CGPoint point = CGPointMake(x.doubleValue, y.doubleValue); + [self performMoves:@[[self moveFrom:@"viewport" + x:frame[0].doubleValue + point.x y:frame[1].doubleValue + point.y duration:0]] + expectedEnd:point]; + [self performMoves:@[[self moveFrom:self.canvas x:point.x - width / 2 y:point.y - height / 2 duration:0]] + expectedEnd:point]; + } + } +} + +- (void)testElementHitpoint +{ + NSArray *bounds = [self measurement][@"canvasBounds"]; + [self performMoves:@[@{@"type": @"pointerMove", @"origin": self.canvas, @"duration": @0}] + expectedEnd:CGPointMake(bounds[0].doubleValue / 2, bounds[1].doubleValue / 2)]; +} + +- (void)testPointerRelativeMovesPreserveOriginSpace +{ + NSDictionary *geometry = [self measurement]; + NSArray *bounds = geometry[@"canvasBounds"]; + NSArray *frame = geometry[@"canvasWindowRect"]; + CGPoint center = CGPointMake(bounds[0].doubleValue / 2, bounds[1].doubleValue / 2); + for (NSDictionary *start in @[ + [self moveFrom:self.canvas x:0 y:0 duration:0], + [self moveFrom:@"viewport" x:frame[0].doubleValue + center.x y:frame[1].doubleValue + center.y duration:0], + ]) { + [self performMoves:@[start, + [self moveFrom:@"pointer" x:0 y:-40 duration:200], + [self moveFrom:@"pointer" x:0 y:-40 duration:200]] + expectedEnd:CGPointMake(center.x, center.y - 80)]; + } +} + +- (void)testMixedOriginsWithinOneTouch +{ + NSDictionary *geometry = [self measurement]; + NSArray *bounds = geometry[@"canvasBounds"]; + NSArray *frame = geometry[@"canvasWindowRect"]; + CGPoint center = CGPointMake(bounds[0].doubleValue / 2, bounds[1].doubleValue / 2); + for (NSNumber *startAtElement in @[@YES, @NO]) { + NSDictionary *start = startAtElement.boolValue + ? [self moveFrom:self.canvas x:0 y:0 duration:0] + : [self moveFrom:@"viewport" x:frame[0].doubleValue + center.x y:frame[1].doubleValue + center.y duration:0]; + NSDictionary *next = startAtElement.boolValue + ? [self moveFrom:@"viewport" x:frame[0].doubleValue + center.x y:frame[1].doubleValue + center.y - 40 duration:200] + : [self moveFrom:self.canvas x:0 y:-40 duration:200]; + [self performMoves:@[start, next, [self moveFrom:@"pointer" x:0 y:-40 duration:200]] + expectedEnd:CGPointMake(center.x, center.y - 80)]; + } +} + +- (void)testScrollRemainsInForegroundAndMovesContent +{ + [self.testedApplication.buttons[@"Scroll"] tap]; + XCUIElement *table = self.testedApplication.tables[@"probe-table"]; + CGFloat before = [[self measurement][@"scrollY"] doubleValue]; + [table fb_scrollDownByNormalizedDistance:0.5]; + XCTAssertEqual(self.testedApplication.state, XCUIApplicationStateRunningForeground); + CGFloat after = [[self measurement][@"scrollY"] doubleValue]; + XCTAssertGreaterThan(after, before + 10); + [table fb_scrollUpByNormalizedDistance:0.5]; + XCTAssertLessThan([[self measurement][@"scrollY"] doubleValue], after - 10); +} + +- (void)testScrollToElementKeepsTheWholeCellVisible +{ + [self.testedApplication.buttons[@"Scroll"] tap]; + XCUIElement *cell = self.testedApplication.cells[@"probe-row-13"]; + NSError *error = nil; + XCTAssertTrue([cell fb_scrollToVisibleWithError:&error], @"%@", error); + XCTAssertEqual(self.testedApplication.state, XCUIApplicationStateRunningForeground); + NSArray *frame = [self measurement][@"canvasWindowRect"]; + XCTAssertTrue(cell.hittable); + XCTAssertGreaterThanOrEqual(CGRectGetMinY(cell.frame), frame[1].doubleValue - 1); + XCTAssertLessThanOrEqual(CGRectGetMaxY(cell.frame), frame[1].doubleValue + frame[3].doubleValue + 1); +} + +@end diff --git a/WebDriverAgentTests/UnitTests/FBMathUtilsTests.m b/WebDriverAgentTests/UnitTests/FBMathUtilsTests.m index cfd3486890..a5f65ef8c1 100644 --- a/WebDriverAgentTests/UnitTests/FBMathUtilsTests.m +++ b/WebDriverAgentTests/UnitTests/FBMathUtilsTests.m @@ -178,4 +178,19 @@ - (void)testDisplayCoordinateOffsets } } +- (void)testPortraitVisibleFrameRotation +{ + CGSize size = CGSizeMake(669, 951); + CGRect clipped = CGRectMake(200, 100, 300, 400); + XCTAssertTrue(CGRectEqualToRect(FBRectFromPortraitCoordinates(clipped, size, UIInterfaceOrientationPortrait), clipped)); + XCTAssertTrue(CGRectEqualToRect(FBRectFromPortraitCoordinates(clipped, size, UIInterfaceOrientationLandscapeLeft), CGRectMake(451, 200, 400, 300))); + XCTAssertTrue(CGRectEqualToRect(FBRectFromPortraitCoordinates(clipped, size, UIInterfaceOrientationLandscapeRight), CGRectMake(100, 169, 400, 300))); + XCTAssertTrue(CGRectEqualToRect(FBRectFromPortraitCoordinates(clipped, size, UIInterfaceOrientationPortraitUpsideDown), CGRectMake(169, 451, 300, 400))); + // Observed on Duo's inner landscape display: keep the clipping in the same + // coordinate space as the element's reported frame. + XCTAssertTrue(CGRectEqualToRect(FBRectFromPortraitCoordinates(CGRectMake(236, 104, 379, 827), size, UIInterfaceOrientationLandscapeLeft), CGRectMake(20, 236, 827, 379))); + XCTAssertTrue(CGRectIsEmpty(FBRectFromPortraitCoordinates(CGRectZero, size, UIInterfaceOrientationLandscapeLeft))); + XCTAssertTrue(CGRectIsNull(FBRectFromPortraitCoordinates(CGRectNull, size, UIInterfaceOrientationLandscapeLeft))); +} + @end From 85dd686bcaf57eb6048618ac4a27259ca3b22a88 Mon Sep 17 00:00:00 2001 From: Kazuaki Matsuo Date: Sat, 3 Oct 2026 01:28:52 -0700 Subject: [PATCH 09/29] remove simulator only logic, temporary implementation --- README.md | 36 +---------------- .../Categories/XCUIDevice+FBHelpers.h | 8 +++- .../Categories/XCUIDevice+FBHelpers.m | 30 ++++++++++---- WebDriverAgentLib/Commands/FBCustomCommands.m | 2 +- .../UnitTests/FBCustomCommandsTests.m | 2 +- docs/multiple-displays.md | 39 +++++++++++++++++++ 6 files changed, 72 insertions(+), 45 deletions(-) create mode 100644 docs/multiple-displays.md diff --git a/README.md b/README.md index 1b42f28852..4992598ecd 100644 --- a/README.md +++ b/README.md @@ -29,40 +29,8 @@ More about how to start WebDriverAgent [here](https://github.com/facebook/WebDri ## Multiple displays -On devices such as iPhone Duo, `GET /wda/screens` lists the available displays. -Set `currentDisplayId` through `POST /session/:sessionId/appium/settings` to select -one of these IDs. Do not assume that the main display is the currently visible -one after folding or unfolding the device. - -The setting selects the display for screenshots, MJPEG frames, and newly started -XCTest screen recordings. MJPEG follows setting changes on the existing -connection; an in-progress recording keeps the display selected when it started. -An unavailable display causes screenshots and recording requests to fail; MJPEG -pauses frame delivery until a valid display is selected. Set `currentDisplayId` -to `null` to restore the main display. WDA does not automatically switch this -setting when the device changes pose. - -XCTest's active-app point lookup only targets the main display. When a secondary -display is selected, WDA uses its active-app fallback instead of that point lookup. -This is not display-specific app filtering: set `defaultActiveApplication` to the -app's bundle ID when multiple foreground apps make the result ambiguous. An app -specified when creating the session retains its existing priority. - -W3C coordinate actions require the separate display-aware actions support tracked -in [#1269](https://github.com/appium/WebDriverAgent/pull/1269). - -### Simulated hinge angle - -On an iPhone Duo simulator, `POST /session/:sessionId/wda/device/hingeAngle` with -`{"angle": 90}` sends a hinge event without operating Device Hub's UI. The angle -must be a finite number from `0` (closed) to `180` (fully open); fractional values -are supported. This uses the simulator's private HID protocol verified with -Xcode 27.1 and is currently limited to model `iPhone19,4`. Other simulators and -physical devices return `unsupported operation`. - -The response confirms event dispatch. Wait for the expected app layout before -continuing, since folding completes asynchronously. The command does not rotate -the device or update `currentDisplayId`; select the desired display separately. +See [Multiple displays](docs/multiple-displays.md) for display selection, capture, +active-app detection, and simulated hinge control. ## Known Issues diff --git a/WebDriverAgentLib/Categories/XCUIDevice+FBHelpers.h b/WebDriverAgentLib/Categories/XCUIDevice+FBHelpers.h index 593717c983..1d2e912271 100644 --- a/WebDriverAgentLib/Categories/XCUIDevice+FBHelpers.h +++ b/WebDriverAgentLib/Categories/XCUIDevice+FBHelpers.h @@ -231,13 +231,17 @@ typedef NS_ENUM(NSUInteger, FBUIInterfaceAppearance) { - (BOOL)fb_clearSimulatedLocation:(NSError **)error; #endif -/** Whether the current runtime is the supported iPhone Duo simulator. */ +/** + Whether the runtime reports an available hinge, permitting an injection attempt. + This does not guarantee that the device accepts simulated hinge events. + */ - (BOOL)fb_supportsSimulatedHingeAngle; /** Sends a simulated hinge angle in degrees (0 = closed, 180 = fully open). This dispatches the event; the display transition completes asynchronously. - Only supported on the iPhone Duo simulator. Does not change currentDisplayId. + Verified on the iPhone Duo simulator; physical-device behavior is unverified. + Does not change currentDisplayId. */ - (BOOL)fb_setSimulatedHingeAngle:(double)angle error:(NSError **)error; diff --git a/WebDriverAgentLib/Categories/XCUIDevice+FBHelpers.m b/WebDriverAgentLib/Categories/XCUIDevice+FBHelpers.m index c9aa2ae8eb..917e0c04fa 100644 --- a/WebDriverAgentLib/Categories/XCUIDevice+FBHelpers.m +++ b/WebDriverAgentLib/Categories/XCUIDevice+FBHelpers.m @@ -25,6 +25,12 @@ #import "FBXCTestDaemonsProxy.h" #import "XCUIDevice.h" +#if !TARGET_OS_TV && !TARGET_OS_WATCH +@protocol FBAngleManager ++ (BOOL)isAvailable; +@end +#endif + static const NSTimeInterval FBHomeButtonCoolOffTime = 1.; static const NSTimeInterval FBScreenLockTimeout = 5.; @@ -479,10 +485,20 @@ - (BOOL)fb_clearSimulatedLocation:(NSError **)error - (BOOL)fb_supportsSimulatedHingeAngle { -#if TARGET_OS_SIMULATOR && !TARGET_OS_TV && !TARGET_OS_WATCH - // This vendor event is specific to Duo's simulated hardware. Do not send it - // to physical devices or assume every device with multiple screens supports it. - return [NSProcessInfo.processInfo.environment[@"SIMULATOR_MODEL_IDENTIFIER"] isEqualToString:@"iPhone19,4"]; +#if !TARGET_OS_TV && !TARGET_OS_WATCH + static Class angleManagerClass; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + if (NULL != dlopen("/System/Library/Frameworks/CoreMotion.framework/CoreMotion", RTLD_LAZY)) { + angleManagerClass = NSClassFromString(@"CMAngleManager"); + } + }); + if (![angleManagerClass respondsToSelector:@selector(isAvailable)]) { + return NO; + } + // Hinge availability permits an injection attempt; it does not guarantee + // that the device accepts the vendor event. Physical devices are unverified. + return [angleManagerClass isAvailable]; #else return NO; #endif @@ -494,9 +510,9 @@ - (BOOL)fb_setSimulatedHingeAngle:(double)angle error:(NSError **)error return [[FBErrorBuilder.builder withDescription:@"Hinge angle must be a finite number between 0 and 180 degrees"] buildError:error]; } if (!self.fb_supportsSimulatedHingeAngle) { - return [[FBErrorBuilder.builder withDescription:@"Simulated hinge angle is only supported on the iPhone Duo simulator"] buildError:error]; + return [[FBErrorBuilder.builder withDescription:@"The device does not report an available hinge"] buildError:error]; } -#if TARGET_OS_SIMULATOR && !TARGET_OS_TV && !TARGET_OS_WATCH +#if !TARGET_OS_TV && !TARGET_OS_WATCH static CFDataRef (*serialize)(CFTypeRef, CFOptionFlags); static CFTypeRef (*createEvent)(CFAllocatorRef, uint64_t, uint32_t, uint32_t, uint32_t, uint8_t *, CFIndex, uint32_t); static CFTypeRef (*createClient)(CFAllocatorRef); @@ -512,7 +528,7 @@ - (BOOL)fb_setSimulatedHingeAngle:(double)angle error:(NSError **)error } }); if (NULL == serialize || NULL == createEvent || NULL == createClient || NULL == dispatchEvent) { - return [[FBErrorBuilder.builder withDescription:@"The simulator runtime does not provide the required IOKit HID APIs"] buildError:error]; + return [[FBErrorBuilder.builder withDescription:@"The runtime does not provide the required IOKit HID APIs"] buildError:error]; } // Matches Device Hub's hinge-slider-control payload (Xcode 27.1). It is an // IOCF binary serialization, not an NSPropertyListSerialization binary plist. diff --git a/WebDriverAgentLib/Commands/FBCustomCommands.m b/WebDriverAgentLib/Commands/FBCustomCommands.m index 700fca0fde..156c96bede 100644 --- a/WebDriverAgentLib/Commands/FBCustomCommands.m +++ b/WebDriverAgentLib/Commands/FBCustomCommands.m @@ -789,7 +789,7 @@ + (NSString *)timeZone } if (!XCUIDevice.sharedDevice.fb_supportsSimulatedHingeAngle) { return FBResponseWithStatus([FBCommandStatus unsupportedOperationErrorWithMessage: - @"Simulated hinge angle is only supported on the iPhone Duo simulator" traceback:nil]); + @"The device does not report an available hinge" traceback:nil]); } NSError *error; if (![XCUIDevice.sharedDevice fb_setSimulatedHingeAngle:[angle doubleValue] error:&error]) { diff --git a/WebDriverAgentTests/UnitTests/FBCustomCommandsTests.m b/WebDriverAgentTests/UnitTests/FBCustomCommandsTests.m index 842c07ceb5..8ec071efcd 100644 --- a/WebDriverAgentTests/UnitTests/FBCustomCommandsTests.m +++ b/WebDriverAgentTests/UnitTests/FBCustomCommandsTests.m @@ -103,7 +103,7 @@ - (void)testHingeAngleRejectsInvalidArguments - (void)testHingeAngleRejectsUnsupportedDevices { - XCTSkipIf(XCUIDevice.sharedDevice.fb_supportsSimulatedHingeAngle, @"Requires a device without a simulated hinge"); + XCTSkipIf(XCUIDevice.sharedDevice.fb_supportsSimulatedHingeAngle, @"Requires a device without an available hinge"); for (NSNumber *angle in @[@0, @90, @180]) { XCTAssertEqualObjects([self hingeResponseWithArguments:@{@"angle": angle}][@"value"][@"error"], @"unsupported operation"); diff --git a/docs/multiple-displays.md b/docs/multiple-displays.md new file mode 100644 index 0000000000..173093a42e --- /dev/null +++ b/docs/multiple-displays.md @@ -0,0 +1,39 @@ +# Multiple displays + +On devices such as iPhone Duo, `GET /wda/screens` lists the available displays. +Set `currentDisplayId` through `POST /session/:sessionId/appium/settings` to select +one of these IDs. Do not assume that the main display is the currently visible +one after folding or unfolding the device. + +The setting selects the display for screenshots, MJPEG frames, and newly started +XCTest screen recordings. MJPEG follows setting changes on the existing +connection; an in-progress recording keeps the display selected when it started. +An unavailable display causes screenshots and recording requests to fail; MJPEG +pauses frame delivery until a valid display is selected. Set `currentDisplayId` +to `null` to restore the main display. WDA does not automatically switch this +setting when the device changes pose. + +The XCTest point lookup used by WDA has no display argument, including in the +Xcode 27.1 interfaces checked for this change. When a secondary display is +selected, WDA uses its active-app fallback instead of that main-display lookup. +This is not display-specific app filtering: set `defaultActiveApplication` to the +app's bundle ID when multiple foreground apps make the result ambiguous. An app +specified when creating the session retains its existing priority. + +W3C coordinate actions require the separate display-aware actions support tracked +in [#1269](https://github.com/appium/WebDriverAgent/pull/1269). + +## Simulated hinge angle + +On an iPhone Duo simulator, `POST /session/:sessionId/wda/device/hingeAngle` with +`{"angle": 90}` sends a hinge event without operating Device Hub's UI. The angle +must be a finite number from `0` (closed) to `180` (fully open); fractional values +are supported. This uses a private HID protocol verified with Xcode 27.1 on the +iPhone Duo simulator. WDA checks runtime hinge availability instead of a fixed +model identifier and permits injection attempts on physical devices as well. +Physical-device behavior is unverified. Devices without an available hinge, +tvOS, and watchOS return `unsupported operation`. + +The response confirms event dispatch, not that the device applied the angle. Wait for the expected app layout before +continuing, since folding completes asynchronously. The command does not rotate +the device or update `currentDisplayId`; select the desired display separately. From db7aa9a1b036a3b41618b927845b2cbf84894d41 Mon Sep 17 00:00:00 2001 From: Kazuaki Matsuo Date: Sat, 3 Oct 2026 01:35:57 -0700 Subject: [PATCH 10/29] add get --- .../Categories/XCUIDevice+FBHelpers.h | 7 ++ .../Categories/XCUIDevice+FBHelpers.m | 84 +++++++++++++++++-- WebDriverAgentLib/Commands/FBCustomCommands.m | 14 ++++ .../UnitTests/FBCustomCommandsTests.m | 29 +++++++ docs/multiple-displays.md | 12 +++ 5 files changed, 139 insertions(+), 7 deletions(-) diff --git a/WebDriverAgentLib/Categories/XCUIDevice+FBHelpers.h b/WebDriverAgentLib/Categories/XCUIDevice+FBHelpers.h index 1d2e912271..8f5e0fd390 100644 --- a/WebDriverAgentLib/Categories/XCUIDevice+FBHelpers.h +++ b/WebDriverAgentLib/Categories/XCUIDevice+FBHelpers.h @@ -237,6 +237,13 @@ typedef NS_ENUM(NSUInteger, FBUIInterfaceAppearance) { */ - (BOOL)fb_supportsSimulatedHingeAngle; +/** + Reads the current hinge angle in degrees, including changes made outside WDA. + Waits up to 5 seconds for a valid reading. Does not change the hinge angle. + @return The angle, or nil with an error if unavailable or reading times out. + */ +- (nullable NSNumber *)fb_getSimulatedHingeAngle:(NSError **)error; + /** Sends a simulated hinge angle in degrees (0 = closed, 180 = fully open). This dispatches the event; the display transition completes asynchronously. diff --git a/WebDriverAgentLib/Categories/XCUIDevice+FBHelpers.m b/WebDriverAgentLib/Categories/XCUIDevice+FBHelpers.m index 917e0c04fa..ee32ee82ab 100644 --- a/WebDriverAgentLib/Categories/XCUIDevice+FBHelpers.m +++ b/WebDriverAgentLib/Categories/XCUIDevice+FBHelpers.m @@ -26,9 +26,29 @@ #import "XCUIDevice.h" #if !TARGET_OS_TV && !TARGET_OS_WATCH +@protocol FBAngle +- (BOOL)isAngleValid; +- (float)angleDegrees; +@end + @protocol FBAngleManager ++ (instancetype)new; + (BOOL)isAvailable; +- (void)startAngleUpdatesToQueue:(NSOperationQueue *)queue handler:(void (^)(id))handler; +- (void)stopAngleUpdates; @end + +static Class FBAngleManagerClass(void) +{ + static Class angleManagerClass; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + if (NULL != dlopen("/System/Library/Frameworks/CoreMotion.framework/CoreMotion", RTLD_LAZY)) { + angleManagerClass = NSClassFromString(@"CMAngleManager"); + } + }); + return angleManagerClass; +} #endif static const NSTimeInterval FBHomeButtonCoolOffTime = 1.; @@ -486,13 +506,7 @@ - (BOOL)fb_clearSimulatedLocation:(NSError **)error - (BOOL)fb_supportsSimulatedHingeAngle { #if !TARGET_OS_TV && !TARGET_OS_WATCH - static Class angleManagerClass; - static dispatch_once_t onceToken; - dispatch_once(&onceToken, ^{ - if (NULL != dlopen("/System/Library/Frameworks/CoreMotion.framework/CoreMotion", RTLD_LAZY)) { - angleManagerClass = NSClassFromString(@"CMAngleManager"); - } - }); + Class angleManagerClass = FBAngleManagerClass(); if (![angleManagerClass respondsToSelector:@selector(isAvailable)]) { return NO; } @@ -504,6 +518,62 @@ - (BOOL)fb_supportsSimulatedHingeAngle #endif } +- (nullable NSNumber *)fb_getSimulatedHingeAngle:(NSError **)error +{ + if (!self.fb_supportsSimulatedHingeAngle) { + [[FBErrorBuilder.builder withDescription:@"The device does not report an available hinge"] buildError:error]; + return nil; + } +#if !TARGET_OS_TV && !TARGET_OS_WATCH + id manager = [FBAngleManagerClass() new]; + if (![manager respondsToSelector:@selector(startAngleUpdatesToQueue:handler:)] + || ![manager respondsToSelector:@selector(stopAngleUpdates)]) { + [[FBErrorBuilder.builder withDescription:@"The runtime does not provide the required hinge angle reading APIs"] buildError:error]; + return nil; + } + NSOperationQueue *queue = [NSOperationQueue new]; + queue.maxConcurrentOperationCount = 1; + NSCondition *condition = [NSCondition new]; + __block NSNumber *reading = nil; + NSNumber *result; + @try { + [manager startAngleUpdatesToQueue:queue handler:^(id angle) { + if (![angle respondsToSelector:@selector(isAngleValid)] + || ![angle respondsToSelector:@selector(angleDegrees)] || !angle.isAngleValid) { + return; + } + float degrees = angle.angleDegrees; + if (!isfinite(degrees)) { + return; + } + [condition lock]; + if (nil == reading) { + reading = @(degrees); + [condition signal]; + } + [condition unlock]; + }]; + NSDate *deadline = [NSDate dateWithTimeIntervalSinceNow:5.0]; + [condition lock]; + while (nil == reading) { + if (![condition waitUntilDate:deadline]) { + break; + } + } + result = reading; + [condition unlock]; + } @finally { + [manager stopAngleUpdates]; + } + if (nil == result) { + [[FBErrorBuilder.builder withDescription:@"Timed out after 5 seconds waiting for a valid hinge angle"] buildError:error]; + } + return result; +#else + return nil; +#endif +} + - (BOOL)fb_setSimulatedHingeAngle:(double)angle error:(NSError **)error { if (!isfinite(angle) || angle < 0 || angle > 180) { diff --git a/WebDriverAgentLib/Commands/FBCustomCommands.m b/WebDriverAgentLib/Commands/FBCustomCommands.m index 156c96bede..08cf756376 100644 --- a/WebDriverAgentLib/Commands/FBCustomCommands.m +++ b/WebDriverAgentLib/Commands/FBCustomCommands.m @@ -74,7 +74,10 @@ + (NSArray *)routes [[FBRoute POST:@"/wda/expectNotification"] respondWithTarget:self action:@selector(handleExpectNotification:)], [[FBRoute POST:@"/wda/siri/activate"] respondWithTarget:self action:@selector(handleActivateSiri:)], [[FBRoute POST:@"/wda/apps/launchUnattached"].withoutSession respondWithTarget:self action:@selector(handleLaunchUnattachedApp:)], + [[FBRoute GET:@"/wda/device/hingeAngle"] respondWithTarget:self action:@selector(handleGetSimulatedHingeAngle:)], + [[FBRoute GET:@"/wda/device/hingeAngle"].withoutSession respondWithTarget:self action:@selector(handleGetSimulatedHingeAngle:)], [[FBRoute POST:@"/wda/device/hingeAngle"] respondWithTarget:self action:@selector(handleSetSimulatedHingeAngle:)], + [[FBRoute POST:@"/wda/device/hingeAngle"].withoutSession respondWithTarget:self action:@selector(handleSetSimulatedHingeAngle:)], [[FBRoute GET:@"/wda/device/info"] respondWithTarget:self action:@selector(handleGetDeviceInfo:)], [[FBRoute POST:@"/wda/resetAppAuth"] respondWithTarget:self action:@selector(handleResetAppAuth:)], [[FBRoute GET:@"/wda/device/info"].withoutSession respondWithTarget:self action:@selector(handleGetDeviceInfo:)], @@ -778,6 +781,17 @@ + (NSString *)timeZone return FBResponseWithObject(result); } ++ (id)handleGetSimulatedHingeAngle:(FBRouteRequest *)request +{ + if (!XCUIDevice.sharedDevice.fb_supportsSimulatedHingeAngle) { + return FBResponseWithStatus([FBCommandStatus unsupportedOperationErrorWithMessage: + @"The device does not report an available hinge" traceback:nil]); + } + NSError *error; + NSNumber *angle = [XCUIDevice.sharedDevice fb_getSimulatedHingeAngle:&error]; + return nil == angle ? FBResponseWithUnknownError(error) : FBResponseWithObject(angle); +} + + (id)handleSetSimulatedHingeAngle:(FBRouteRequest *)request { id angle = request.arguments[@"angle"]; diff --git a/WebDriverAgentTests/UnitTests/FBCustomCommandsTests.m b/WebDriverAgentTests/UnitTests/FBCustomCommandsTests.m index 8ec071efcd..ff9a8b19ca 100644 --- a/WebDriverAgentTests/UnitTests/FBCustomCommandsTests.m +++ b/WebDriverAgentTests/UnitTests/FBCustomCommandsTests.m @@ -21,6 +21,7 @@ @interface FBCustomCommands (FBWDATestable) + (id)handleKeyboardInput:(FBRouteRequest *)request; ++ (id)handleGetSimulatedHingeAngle:(FBRouteRequest *)request; + (id)handleSetSimulatedHingeAngle:(FBRouteRequest *)request; @end @@ -92,6 +93,34 @@ - (NSDictionary *)hingeResponseWithArguments:(NSDictionary *)arguments return [NSJSONSerialization JSONObjectWithData:response.responseData options:0 error:nil]; } +- (void)testHingeAngleReading +{ + NSError *error = nil; + NSNumber *angle = [XCUIDevice.sharedDevice fb_getSimulatedHingeAngle:&error]; + BOOL available = XCUIDevice.sharedDevice.fb_supportsSimulatedHingeAngle; + if (available) { + XCTAssertNotNil(angle); + XCTAssertNil(error); + XCTAssertTrue(isfinite(angle.doubleValue)); + } else { + XCTAssertNil(angle); + XCTAssertNotNil(error); + } + + FBRouteRequest *request = [FBRouteRequest routeRequestWithURL:[NSURL URLWithString:@"http://localhost:8100/"] + parameters:@{} + arguments:@{}]; + RouteResponse *response = [RouteResponse new]; + [[FBCustomCommands handleGetSimulatedHingeAngle:request] dispatchWithResponse:response]; + NSDictionary *body = [NSJSONSerialization JSONObjectWithData:response.responseData options:0 error:nil]; + if (available) { + XCTAssertTrue([body[@"value"] isKindOfClass:NSNumber.class]); + XCTAssertTrue(isfinite([body[@"value"] doubleValue])); + } else { + XCTAssertEqualObjects(body[@"value"][@"error"], @"unsupported operation"); + } +} + - (void)testHingeAngleRejectsInvalidArguments { for (id value in @[@(-1), @181, @YES, @"90", NSNull.null, @(NAN), @(INFINITY)]) { diff --git a/docs/multiple-displays.md b/docs/multiple-displays.md index 173093a42e..44e5a97da4 100644 --- a/docs/multiple-displays.md +++ b/docs/multiple-displays.md @@ -37,3 +37,15 @@ tvOS, and watchOS return `unsupported operation`. The response confirms event dispatch, not that the device applied the angle. Wait for the expected app layout before continuing, since folding completes asynchronously. The command does not rotate the device or update `currentDisplayId`; select the desired display separately. + +`GET /session/:sessionId/wda/device/hingeAngle` reads the current angle in degrees +and returns a numeric value, for example `{"value": 90.5}`. It reads CoreMotion +rather than remembering the last angle sent by WDA, so changes made outside WDA +are reflected too. The request waits up to five seconds for a valid reading and +returns an error if none arrives. Devices without an available hinge return +`unsupported operation`. A reading during a fold may reflect an intermediate +angle; it does not wait for the transition to finish. + +Both hinge angle endpoints are also available without a session: +`GET /wda/device/hingeAngle` and `POST /wda/device/hingeAngle`. They operate on +the device and do not require creating an Appium session first. From f0b161752e64362051cbf46e2124cfcfc1bcb2b6 Mon Sep 17 00:00:00 2001 From: Kazuaki Matsuo Date: Sat, 3 Oct 2026 01:43:10 -0700 Subject: [PATCH 11/29] refactor a bit --- WebDriverAgent.xcodeproj/project.pbxproj | 16 ++ .../Categories/XCUIDevice+FBHelpers.h | 21 -- .../Categories/XCUIDevice+FBHelpers.m | 157 --------------- .../Categories/XCUIDevice+FBHinge.h | 38 ++++ .../Categories/XCUIDevice+FBHinge.m | 186 ++++++++++++++++++ WebDriverAgentLib/Commands/FBCustomCommands.m | 1 + WebDriverAgentLib/WebDriverAgentLib.h | 1 + .../UnitTests/FBCustomCommandsTests.m | 2 +- 8 files changed, 243 insertions(+), 179 deletions(-) create mode 100644 WebDriverAgentLib/Categories/XCUIDevice+FBHinge.h create mode 100644 WebDriverAgentLib/Categories/XCUIDevice+FBHinge.m diff --git a/WebDriverAgent.xcodeproj/project.pbxproj b/WebDriverAgent.xcodeproj/project.pbxproj index 593981fc1d..2c9da2fd99 100644 --- a/WebDriverAgent.xcodeproj/project.pbxproj +++ b/WebDriverAgent.xcodeproj/project.pbxproj @@ -245,6 +245,7 @@ 57C72DA18039FED310967F8F /* XCTSourceCodeLocation.h in Headers */ = {isa = PBXBuildFile; fileRef = 81BB54ECBFB5B7B680BB5D4F /* XCTSourceCodeLocation.h */; }; 57EAF450031647694B6491F8 /* FBConfiguration.m in Sources */ = {isa = PBXBuildFile; fileRef = EE9B76A21CF7A43900275851 /* FBConfiguration.m */; }; 583AD004FF71CAA305E6DEAF /* XCUIDevice+FBHelpers.m in Sources */ = {isa = PBXBuildFile; fileRef = AD6C26971CF2481700F8B5FF /* XCUIDevice+FBHelpers.m */; }; + 0460879AD6034130AE0A3DEC /* XCUIDevice+FBHinge.m in Sources */ = {isa = PBXBuildFile; fileRef = 8C8E6FD103A84C908F5746EA /* XCUIDevice+FBHinge.m */; }; 587AA46196BC2C46555C8E44 /* XCUIButtonConsole.h in Headers */ = {isa = PBXBuildFile; fileRef = 1E09842154A44874C4E9CA01 /* XCUIButtonConsole.h */; settings = {ATTRIBUTES = (Public, ); }; }; 588F51093FD2A5597366C282 /* XCUIElementEventTarget-Protocol.h in Headers */ = {isa = PBXBuildFile; fileRef = 0DC62BF635704E9C72AF533E /* XCUIElementEventTarget-Protocol.h */; }; 5967912A40E807CDBE87E4B2 /* XCTReportingSessionIssueReporter-Protocol.h in Headers */ = {isa = PBXBuildFile; fileRef = 525D62A58488A52AA1BFE94A /* XCTReportingSessionIssueReporter-Protocol.h */; settings = {ATTRIBUTES = (Public, ); }; }; @@ -327,6 +328,7 @@ 641EE60A2240C5CA00173FCB /* FBLogger.m in Sources */ = {isa = PBXBuildFile; fileRef = EE9B76A41CF7A43900275851 /* FBLogger.m */; }; 641EE60B2240C5CA00173FCB /* FBCustomCommands.m in Sources */ = {isa = PBXBuildFile; fileRef = EE9AB7531CAEDF0C008C271F /* FBCustomCommands.m */; }; 641EE60C2240C5CA00173FCB /* XCUIDevice+FBHelpers.m in Sources */ = {isa = PBXBuildFile; fileRef = AD6C26971CF2481700F8B5FF /* XCUIDevice+FBHelpers.m */; }; + 181F0DBB6D28418680D52128 /* XCUIDevice+FBHinge.m in Sources */ = {isa = PBXBuildFile; fileRef = 8C8E6FD103A84C908F5746EA /* XCUIDevice+FBHinge.m */; }; 641EE60D2240C5CA00173FCB /* XCTestPrivateSymbols.m in Sources */ = {isa = PBXBuildFile; fileRef = EE6B64FC1D0F86EF00E85F5D /* XCTestPrivateSymbols.m */; }; 641EE60E2240C5CA00173FCB /* XCUIElement+FBTyping.m in Sources */ = {isa = PBXBuildFile; fileRef = AD76723C1D6B7CC000610457 /* XCUIElement+FBTyping.m */; }; 641EE60F2240C5CA00173FCB /* XCUIElement+FBAccessibility.m in Sources */ = {isa = PBXBuildFile; fileRef = EE9AB7461CAEDF0C008C271F /* XCUIElement+FBAccessibility.m */; }; @@ -433,6 +435,7 @@ 641EE6BF2240C5CA00173FCB /* FBKeyboard.h in Headers */ = {isa = PBXBuildFile; fileRef = EE3A18641CDE734B00DE4205 /* FBKeyboard.h */; settings = {ATTRIBUTES = (Public, ); }; }; 641EE6C02240C5CA00173FCB /* XCUIApplication+FBHelpers.h in Headers */ = {isa = PBXBuildFile; fileRef = AD6C269A1CF2494200F8B5FF /* XCUIApplication+FBHelpers.h */; settings = {ATTRIBUTES = (Public, ); }; }; 641EE6C22240C5CA00173FCB /* XCUIDevice+FBHelpers.h in Headers */ = {isa = PBXBuildFile; fileRef = AD6C26961CF2481700F8B5FF /* XCUIDevice+FBHelpers.h */; settings = {ATTRIBUTES = (Public, ); }; }; + 95EF6989BCBE4C548C26B140 /* XCUIDevice+FBHinge.h in Headers */ = {isa = PBXBuildFile; fileRef = E7536267DE094076A4EEE1D7 /* XCUIDevice+FBHinge.h */; settings = {ATTRIBUTES = (Public, ); }; }; 641EE6C32240C5CA00173FCB /* FBClassChainQueryParser.h in Headers */ = {isa = PBXBuildFile; fileRef = 71A7EAF71E224648001DA4F2 /* FBClassChainQueryParser.h */; }; 641EE6C42240C5CA00173FCB /* FBMacros.h in Headers */ = {isa = PBXBuildFile; fileRef = EE9B76A51CF7A43900275851 /* FBMacros.h */; settings = {ATTRIBUTES = (Public, ); }; }; 641EE6C52240C5CA00173FCB /* XCTestExpectationDelegate-Protocol.h in Headers */ = {isa = PBXBuildFile; fileRef = EE35ACD91E3B77D600A02D78 /* XCTestExpectationDelegate-Protocol.h */; settings = {ATTRIBUTES = (Public, ); }; }; @@ -887,7 +890,9 @@ AD6C26941CF2379700F8B5FF /* FBAlert.h in Headers */ = {isa = PBXBuildFile; fileRef = AD6C26921CF2379700F8B5FF /* FBAlert.h */; settings = {ATTRIBUTES = (Public, ); }; }; AD6C26951CF2379700F8B5FF /* FBAlert.m in Sources */ = {isa = PBXBuildFile; fileRef = AD6C26931CF2379700F8B5FF /* FBAlert.m */; }; AD6C26981CF2481700F8B5FF /* XCUIDevice+FBHelpers.h in Headers */ = {isa = PBXBuildFile; fileRef = AD6C26961CF2481700F8B5FF /* XCUIDevice+FBHelpers.h */; settings = {ATTRIBUTES = (Public, ); }; }; + 210AB789D14D40308B764DD0 /* XCUIDevice+FBHinge.h in Headers */ = {isa = PBXBuildFile; fileRef = E7536267DE094076A4EEE1D7 /* XCUIDevice+FBHinge.h */; settings = {ATTRIBUTES = (Public, ); }; }; AD6C26991CF2481700F8B5FF /* XCUIDevice+FBHelpers.m in Sources */ = {isa = PBXBuildFile; fileRef = AD6C26971CF2481700F8B5FF /* XCUIDevice+FBHelpers.m */; }; + 25C9BDFC65564D78A70231B4 /* XCUIDevice+FBHinge.m in Sources */ = {isa = PBXBuildFile; fileRef = 8C8E6FD103A84C908F5746EA /* XCUIDevice+FBHinge.m */; }; AD6C269C1CF2494200F8B5FF /* XCUIApplication+FBHelpers.h in Headers */ = {isa = PBXBuildFile; fileRef = AD6C269A1CF2494200F8B5FF /* XCUIApplication+FBHelpers.h */; settings = {ATTRIBUTES = (Public, ); }; }; AD6C269D1CF2494200F8B5FF /* XCUIApplication+FBHelpers.m in Sources */ = {isa = PBXBuildFile; fileRef = AD6C269B1CF2494200F8B5FF /* XCUIApplication+FBHelpers.m */; }; AD764F10E2F00090AEB802F2 /* XCUIElement+FBIsVisible.m in Sources */ = {isa = PBXBuildFile; fileRef = EE9AB7481CAEDF0C008C271F /* XCUIElement+FBIsVisible.m */; }; @@ -987,6 +992,7 @@ D1172A7F53F89B78D8324A13 /* RouteRequest.m in Sources */ = {isa = PBXBuildFile; fileRef = 374BB11AE4E4243BCA444CF8 /* RouteRequest.m */; }; D1212325FDE77754EE503755 /* FBRouteRequest.m in Sources */ = {isa = PBXBuildFile; fileRef = EE9AB7881CAEDF0C008C271F /* FBRouteRequest.m */; }; D142183578036F5FD5B9880B /* XCUIDevice+FBHelpers.h in Headers */ = {isa = PBXBuildFile; fileRef = AD6C26961CF2481700F8B5FF /* XCUIDevice+FBHelpers.h */; }; + 2E71304F39B64EDC98EE3974 /* XCUIDevice+FBHinge.h in Headers */ = {isa = PBXBuildFile; fileRef = E7536267DE094076A4EEE1D7 /* XCUIDevice+FBHinge.h */; }; D14718959F7D46949859EEE6 /* FBDebugLogDelegateDecorator.m in Sources */ = {isa = PBXBuildFile; fileRef = EE7E27191D06C69F001BEC7B /* FBDebugLogDelegateDecorator.m */; }; D1DE06DF54156F744E364B5F /* FBDebugLogDelegateDecorator.h in Headers */ = {isa = PBXBuildFile; fileRef = EE7E27181D06C69F001BEC7B /* FBDebugLogDelegateDecorator.h */; }; D203F622C7C0D968F3BD2AAF /* XCUIPlatformApplicationServicesProviding-Protocol.h in Headers */ = {isa = PBXBuildFile; fileRef = 3BA71BCF1FDA482419CA8596 /* XCUIPlatformApplicationServicesProviding-Protocol.h */; }; @@ -1760,7 +1766,9 @@ AD6C26921CF2379700F8B5FF /* FBAlert.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = FBAlert.h; path = WebDriverAgentLib/FBAlert.h; sourceTree = SOURCE_ROOT; }; AD6C26931CF2379700F8B5FF /* FBAlert.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; lineEnding = 0; name = FBAlert.m; path = WebDriverAgentLib/FBAlert.m; sourceTree = SOURCE_ROOT; xcLanguageSpecificationIdentifier = xcode.lang.objc; }; AD6C26961CF2481700F8B5FF /* XCUIDevice+FBHelpers.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "XCUIDevice+FBHelpers.h"; sourceTree = ""; }; + E7536267DE094076A4EEE1D7 /* XCUIDevice+FBHinge.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "XCUIDevice+FBHinge.h"; sourceTree = ""; }; AD6C26971CF2481700F8B5FF /* XCUIDevice+FBHelpers.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = "XCUIDevice+FBHelpers.m"; sourceTree = ""; }; + 8C8E6FD103A84C908F5746EA /* XCUIDevice+FBHinge.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = "XCUIDevice+FBHinge.m"; sourceTree = ""; }; AD6C269A1CF2494200F8B5FF /* XCUIApplication+FBHelpers.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "XCUIApplication+FBHelpers.h"; sourceTree = ""; }; AD6C269B1CF2494200F8B5FF /* XCUIApplication+FBHelpers.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = "XCUIApplication+FBHelpers.m"; sourceTree = ""; }; AD76723B1D6B7CC000610457 /* XCUIElement+FBTyping.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "XCUIElement+FBTyping.h"; sourceTree = ""; }; @@ -2415,7 +2423,9 @@ EEDFE11F1D9C06F800E6FFE5 /* XCUIDevice+FBHealthCheck.h */, EEDFE1201D9C06F800E6FFE5 /* XCUIDevice+FBHealthCheck.m */, AD6C26961CF2481700F8B5FF /* XCUIDevice+FBHelpers.h */, + E7536267DE094076A4EEE1D7 /* XCUIDevice+FBHinge.h */, AD6C26971CF2481700F8B5FF /* XCUIDevice+FBHelpers.m */, + 8C8E6FD103A84C908F5746EA /* XCUIDevice+FBHinge.m */, A1B2C3D41F001A00A1B0001 /* XCUIDevice+FBVoiceOver.h */, A1B2C3D41F001A00A1B0002 /* XCUIDevice+FBVoiceOver.m */, EEE3763D1D59F81400ED88DD /* XCUIDevice+FBRotation.h */, @@ -3178,6 +3188,7 @@ 641EE6C02240C5CA00173FCB /* XCUIApplication+FBHelpers.h in Headers */, 714EAA0E2673FDFE005C5B47 /* FBCapabilities.h in Headers */, 641EE6C22240C5CA00173FCB /* XCUIDevice+FBHelpers.h in Headers */, + 95EF6989BCBE4C548C26B140 /* XCUIDevice+FBHinge.h in Headers */, 71D3B3D6267FC7260076473D /* XCUIElement+FBResolve.h in Headers */, 641EE6C32240C5CA00173FCB /* FBClassChainQueryParser.h in Headers */, 641EE6C42240C5CA00173FCB /* FBMacros.h in Headers */, @@ -3489,6 +3500,7 @@ 81231F08969740B0C8C67A30 /* XCUIApplication+FBHelpers.h in Headers */, 9AA0E22033281B4EAA78BC51 /* FBCapabilities.h in Headers */, D142183578036F5FD5B9880B /* XCUIDevice+FBHelpers.h in Headers */, + 2E71304F39B64EDC98EE3974 /* XCUIDevice+FBHinge.h in Headers */, 09D97BD46764FD8149747723 /* XCUIElement+FBResolve.h in Headers */, 6BF3F57247F59B81B8BCEE7B /* FBClassChainQueryParser.h in Headers */, 8AAA33A5B1943B667B0DB05E /* FBMacros.h in Headers */, @@ -3756,6 +3768,7 @@ AD6C269C1CF2494200F8B5FF /* XCUIApplication+FBHelpers.h in Headers */, 714D88CC2733FB970074A925 /* FBXMLGenerationOptions.h in Headers */, AD6C26981CF2481700F8B5FF /* XCUIDevice+FBHelpers.h in Headers */, + 210AB789D14D40308B764DD0 /* XCUIDevice+FBHinge.h in Headers */, 71A7EAF91E224648001DA4F2 /* FBClassChainQueryParser.h in Headers */, EE9B76AA1CF7A43900275851 /* FBMacros.h in Headers */, C8FB547922D4C1FC00B69954 /* FBUnattachedAppLauncher.h in Headers */, @@ -4513,6 +4526,7 @@ 641EE60B2240C5CA00173FCB /* FBCustomCommands.m in Sources */, 71BB58E42B9631F100CB9BFE /* FBScreenRecordingPromise.m in Sources */, 641EE60C2240C5CA00173FCB /* XCUIDevice+FBHelpers.m in Sources */, + 181F0DBB6D28418680D52128 /* XCUIDevice+FBHinge.m in Sources */, 641EE60D2240C5CA00173FCB /* XCTestPrivateSymbols.m in Sources */, 641EE60E2240C5CA00173FCB /* XCUIElement+FBTyping.m in Sources */, 641EE60F2240C5CA00173FCB /* XCUIElement+FBAccessibility.m in Sources */, @@ -4668,6 +4682,7 @@ EDC993E0858F89F73D8E9DF8 /* FBCustomCommands.m in Sources */, 45033CF6BD35C16E34BF76CC /* FBScreenRecordingPromise.m in Sources */, 583AD004FF71CAA305E6DEAF /* XCUIDevice+FBHelpers.m in Sources */, + 0460879AD6034130AE0A3DEC /* XCUIDevice+FBHinge.m in Sources */, 192CACF49EC3825BA6D2EF4F /* XCTestPrivateSymbols.m in Sources */, 53E0F0F38E7847065810FF41 /* XCUIElement+FBTyping.m in Sources */, A5353D849D394BC630839E59 /* XCUIElement+FBAccessibility.m in Sources */, @@ -4787,6 +4802,7 @@ EE9B76A91CF7A43900275851 /* FBLogger.m in Sources */, EE158ABB1CBD456F00A3E3F0 /* FBCustomCommands.m in Sources */, AD6C26991CF2481700F8B5FF /* XCUIDevice+FBHelpers.m in Sources */, + 25C9BDFC65564D78A70231B4 /* XCUIDevice+FBHinge.m in Sources */, 716C9E0227315EFF005AD475 /* XCUIApplication+FBUIInterruptions.m in Sources */, EE6B64FE1D0F86EF00E85F5D /* XCTestPrivateSymbols.m in Sources */, AD76723E1D6B7CC000610457 /* XCUIElement+FBTyping.m in Sources */, diff --git a/WebDriverAgentLib/Categories/XCUIDevice+FBHelpers.h b/WebDriverAgentLib/Categories/XCUIDevice+FBHelpers.h index 8f5e0fd390..761571cdea 100644 --- a/WebDriverAgentLib/Categories/XCUIDevice+FBHelpers.h +++ b/WebDriverAgentLib/Categories/XCUIDevice+FBHelpers.h @@ -231,27 +231,6 @@ typedef NS_ENUM(NSUInteger, FBUIInterfaceAppearance) { - (BOOL)fb_clearSimulatedLocation:(NSError **)error; #endif -/** - Whether the runtime reports an available hinge, permitting an injection attempt. - This does not guarantee that the device accepts simulated hinge events. - */ -- (BOOL)fb_supportsSimulatedHingeAngle; - -/** - Reads the current hinge angle in degrees, including changes made outside WDA. - Waits up to 5 seconds for a valid reading. Does not change the hinge angle. - @return The angle, or nil with an error if unavailable or reading times out. - */ -- (nullable NSNumber *)fb_getSimulatedHingeAngle:(NSError **)error; - -/** - Sends a simulated hinge angle in degrees (0 = closed, 180 = fully open). - This dispatches the event; the display transition completes asynchronously. - Verified on the iPhone Duo simulator; physical-device behavior is unverified. - Does not change currentDisplayId. - */ -- (BOOL)fb_setSimulatedHingeAngle:(double)angle error:(NSError **)error; - @end NS_ASSUME_NONNULL_END diff --git a/WebDriverAgentLib/Categories/XCUIDevice+FBHelpers.m b/WebDriverAgentLib/Categories/XCUIDevice+FBHelpers.m index ee32ee82ab..65b9785c39 100644 --- a/WebDriverAgentLib/Categories/XCUIDevice+FBHelpers.m +++ b/WebDriverAgentLib/Categories/XCUIDevice+FBHelpers.m @@ -12,8 +12,6 @@ #import #include #import -#import -#import #import "FBErrorBuilder.h" #import "FBImageUtils.h" @@ -25,32 +23,6 @@ #import "FBXCTestDaemonsProxy.h" #import "XCUIDevice.h" -#if !TARGET_OS_TV && !TARGET_OS_WATCH -@protocol FBAngle -- (BOOL)isAngleValid; -- (float)angleDegrees; -@end - -@protocol FBAngleManager -+ (instancetype)new; -+ (BOOL)isAvailable; -- (void)startAngleUpdatesToQueue:(NSOperationQueue *)queue handler:(void (^)(id))handler; -- (void)stopAngleUpdates; -@end - -static Class FBAngleManagerClass(void) -{ - static Class angleManagerClass; - static dispatch_once_t onceToken; - dispatch_once(&onceToken, ^{ - if (NULL != dlopen("/System/Library/Frameworks/CoreMotion.framework/CoreMotion", RTLD_LAZY)) { - angleManagerClass = NSClassFromString(@"CMAngleManager"); - } - }); - return angleManagerClass; -} -#endif - static const NSTimeInterval FBHomeButtonCoolOffTime = 1.; static const NSTimeInterval FBScreenLockTimeout = 5.; @@ -503,133 +475,4 @@ - (BOOL)fb_clearSimulatedLocation:(NSError **)error } #endif -- (BOOL)fb_supportsSimulatedHingeAngle -{ -#if !TARGET_OS_TV && !TARGET_OS_WATCH - Class angleManagerClass = FBAngleManagerClass(); - if (![angleManagerClass respondsToSelector:@selector(isAvailable)]) { - return NO; - } - // Hinge availability permits an injection attempt; it does not guarantee - // that the device accepts the vendor event. Physical devices are unverified. - return [angleManagerClass isAvailable]; -#else - return NO; -#endif -} - -- (nullable NSNumber *)fb_getSimulatedHingeAngle:(NSError **)error -{ - if (!self.fb_supportsSimulatedHingeAngle) { - [[FBErrorBuilder.builder withDescription:@"The device does not report an available hinge"] buildError:error]; - return nil; - } -#if !TARGET_OS_TV && !TARGET_OS_WATCH - id manager = [FBAngleManagerClass() new]; - if (![manager respondsToSelector:@selector(startAngleUpdatesToQueue:handler:)] - || ![manager respondsToSelector:@selector(stopAngleUpdates)]) { - [[FBErrorBuilder.builder withDescription:@"The runtime does not provide the required hinge angle reading APIs"] buildError:error]; - return nil; - } - NSOperationQueue *queue = [NSOperationQueue new]; - queue.maxConcurrentOperationCount = 1; - NSCondition *condition = [NSCondition new]; - __block NSNumber *reading = nil; - NSNumber *result; - @try { - [manager startAngleUpdatesToQueue:queue handler:^(id angle) { - if (![angle respondsToSelector:@selector(isAngleValid)] - || ![angle respondsToSelector:@selector(angleDegrees)] || !angle.isAngleValid) { - return; - } - float degrees = angle.angleDegrees; - if (!isfinite(degrees)) { - return; - } - [condition lock]; - if (nil == reading) { - reading = @(degrees); - [condition signal]; - } - [condition unlock]; - }]; - NSDate *deadline = [NSDate dateWithTimeIntervalSinceNow:5.0]; - [condition lock]; - while (nil == reading) { - if (![condition waitUntilDate:deadline]) { - break; - } - } - result = reading; - [condition unlock]; - } @finally { - [manager stopAngleUpdates]; - } - if (nil == result) { - [[FBErrorBuilder.builder withDescription:@"Timed out after 5 seconds waiting for a valid hinge angle"] buildError:error]; - } - return result; -#else - return nil; -#endif -} - -- (BOOL)fb_setSimulatedHingeAngle:(double)angle error:(NSError **)error -{ - if (!isfinite(angle) || angle < 0 || angle > 180) { - return [[FBErrorBuilder.builder withDescription:@"Hinge angle must be a finite number between 0 and 180 degrees"] buildError:error]; - } - if (!self.fb_supportsSimulatedHingeAngle) { - return [[FBErrorBuilder.builder withDescription:@"The device does not report an available hinge"] buildError:error]; - } -#if !TARGET_OS_TV && !TARGET_OS_WATCH - static CFDataRef (*serialize)(CFTypeRef, CFOptionFlags); - static CFTypeRef (*createEvent)(CFAllocatorRef, uint64_t, uint32_t, uint32_t, uint32_t, uint8_t *, CFIndex, uint32_t); - static CFTypeRef (*createClient)(CFAllocatorRef); - static void (*dispatchEvent)(CFTypeRef, CFTypeRef); - static dispatch_once_t onceToken; - dispatch_once(&onceToken, ^{ - void *handle = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_LAZY); - if (NULL != handle) { - serialize = (CFDataRef (*)(CFTypeRef, CFOptionFlags))dlsym(handle, "IOCFSerialize"); - createEvent = (CFTypeRef (*)(CFAllocatorRef, uint64_t, uint32_t, uint32_t, uint32_t, uint8_t *, CFIndex, uint32_t))dlsym(handle, "IOHIDEventCreateVendorDefinedEvent"); - createClient = (CFTypeRef (*)(CFAllocatorRef))dlsym(handle, "IOHIDEventSystemClientCreate"); - dispatchEvent = (void (*)(CFTypeRef, CFTypeRef))dlsym(handle, "IOHIDEventSystemClientDispatchEvent"); - } - }); - if (NULL == serialize || NULL == createEvent || NULL == createClient || NULL == dispatchEvent) { - return [[FBErrorBuilder.builder withDescription:@"The runtime does not provide the required IOKit HID APIs"] buildError:error]; - } - // Matches Device Hub's hinge-slider-control payload (Xcode 27.1). It is an - // IOCF binary serialization, not an NSPropertyListSerialization binary plist. - NSDictionary *payload = @{ - @"provider": @"com.apple.Virtualization.VirtualMachines", - @"source": @"hinge-slider-control", - @"type": @"range", - @"value": @(angle), - }; - CFDataRef data = serialize((__bridge CFTypeRef)payload, 1); - if (NULL == data) { - return [[FBErrorBuilder.builder withDescription:@"Cannot serialize the simulated hinge event"] buildError:error]; - } - CFTypeRef event = createEvent(kCFAllocatorDefault, mach_absolute_time(), 0xff61, 0x5b, 0, - (uint8_t *)CFDataGetBytePtr(data), CFDataGetLength(data), 0); - CFRelease(data); - CFTypeRef client = createClient(kCFAllocatorDefault); - BOOL canDispatch = NULL != event && NULL != client; - if (canDispatch) { - dispatchEvent(client, event); - } - if (NULL != event) { - CFRelease(event); - } - if (NULL != client) { - CFRelease(client); - } - return canDispatch || [[FBErrorBuilder.builder withDescription:@"Cannot create the simulated hinge HID event or client"] buildError:error]; -#else - return NO; -#endif -} - @end diff --git a/WebDriverAgentLib/Categories/XCUIDevice+FBHinge.h b/WebDriverAgentLib/Categories/XCUIDevice+FBHinge.h new file mode 100644 index 0000000000..7cc5e5d9bb --- /dev/null +++ b/WebDriverAgentLib/Categories/XCUIDevice+FBHinge.h @@ -0,0 +1,38 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface XCUIDevice (FBHinge) + +/** + Whether the runtime reports an available hinge, permitting an injection attempt. + This does not guarantee that the device accepts simulated hinge events. + */ +- (BOOL)fb_supportsSimulatedHingeAngle; + +/** + Reads the current hinge angle in degrees, including changes made outside WDA. + Waits up to 5 seconds for a valid reading. Does not change the hinge angle. + @return The angle, or nil with an error if unavailable or reading times out. + */ +- (nullable NSNumber *)fb_getSimulatedHingeAngle:(NSError **)error; + +/** + Sends a simulated hinge angle in degrees (0 = closed, 180 = fully open). + This dispatches the event; the display transition completes asynchronously. + Verified on the iPhone Duo simulator; physical-device behavior is unverified. + Does not change currentDisplayId. + */ +- (BOOL)fb_setSimulatedHingeAngle:(double)angle error:(NSError **)error; + +@end + +NS_ASSUME_NONNULL_END diff --git a/WebDriverAgentLib/Categories/XCUIDevice+FBHinge.m b/WebDriverAgentLib/Categories/XCUIDevice+FBHinge.m new file mode 100644 index 0000000000..35881e6eda --- /dev/null +++ b/WebDriverAgentLib/Categories/XCUIDevice+FBHinge.m @@ -0,0 +1,186 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#import "XCUIDevice+FBHinge.h" + +#import +#import +#import + +#import "FBErrorBuilder.h" + +#if !TARGET_OS_TV && !TARGET_OS_WATCH +typedef CFDataRef (*FBIOCFSerialize)(CFTypeRef object, CFOptionFlags options); +typedef CFTypeRef (*FBIOHIDEventCreateVendorDefinedEvent)(CFAllocatorRef allocator, + uint64_t timestamp, + uint32_t usagePage, + uint32_t usage, + uint32_t version, + uint8_t *data, + CFIndex length, + uint32_t options); +typedef CFTypeRef (*FBIOHIDEventSystemClientCreate)(CFAllocatorRef allocator); +typedef void (*FBIOHIDEventSystemClientDispatchEvent)(CFTypeRef client, CFTypeRef event); + +@protocol FBAngle +- (BOOL)isAngleValid; +- (float)angleDegrees; +@end + +@protocol FBAngleManager ++ (instancetype)new; ++ (BOOL)isAvailable; +- (void)startAngleUpdatesToQueue:(NSOperationQueue *)queue handler:(void (^)(id))handler; +- (void)stopAngleUpdates; +@end + +static Class FBAngleManagerClass(void) +{ + static Class angleManagerClass; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + if (NULL != dlopen("/System/Library/Frameworks/CoreMotion.framework/CoreMotion", RTLD_LAZY)) { + angleManagerClass = NSClassFromString(@"CMAngleManager"); + } + }); + return angleManagerClass; +} +#endif + +@implementation XCUIDevice (FBHinge) + +- (BOOL)fb_supportsSimulatedHingeAngle +{ +#if !TARGET_OS_TV && !TARGET_OS_WATCH + Class angleManagerClass = FBAngleManagerClass(); + if (![angleManagerClass respondsToSelector:@selector(isAvailable)]) { + return NO; + } + // Hinge availability permits an injection attempt; it does not guarantee + // that the device accepts the vendor event. Physical devices are unverified. + return [angleManagerClass isAvailable]; +#else + return NO; +#endif +} + +- (nullable NSNumber *)fb_getSimulatedHingeAngle:(NSError **)error +{ + if (!self.fb_supportsSimulatedHingeAngle) { + [[FBErrorBuilder.builder withDescription:@"The device does not report an available hinge"] buildError:error]; + return nil; + } +#if !TARGET_OS_TV && !TARGET_OS_WATCH + id manager = [FBAngleManagerClass() new]; + if (![manager respondsToSelector:@selector(startAngleUpdatesToQueue:handler:)] + || ![manager respondsToSelector:@selector(stopAngleUpdates)]) { + [[FBErrorBuilder.builder withDescription:@"The runtime does not provide the required hinge angle reading APIs"] buildError:error]; + return nil; + } + NSOperationQueue *queue = [NSOperationQueue new]; + queue.maxConcurrentOperationCount = 1; + NSCondition *condition = [NSCondition new]; + __block NSNumber *reading = nil; + NSNumber *result; + @try { + [manager startAngleUpdatesToQueue:queue handler:^(id angle) { + if (![angle respondsToSelector:@selector(isAngleValid)] + || ![angle respondsToSelector:@selector(angleDegrees)] || !angle.isAngleValid) { + return; + } + float degrees = angle.angleDegrees; + if (!isfinite(degrees)) { + return; + } + [condition lock]; + if (nil == reading) { + reading = @(degrees); + [condition signal]; + } + [condition unlock]; + }]; + NSDate *deadline = [NSDate dateWithTimeIntervalSinceNow:5.0]; + [condition lock]; + while (nil == reading) { + if (![condition waitUntilDate:deadline]) { + break; + } + } + result = reading; + [condition unlock]; + } @finally { + [manager stopAngleUpdates]; + } + if (nil == result) { + [[FBErrorBuilder.builder withDescription:@"Timed out after 5 seconds waiting for a valid hinge angle"] buildError:error]; + } + return result; +#else + return nil; +#endif +} + +- (BOOL)fb_setSimulatedHingeAngle:(double)angle error:(NSError **)error +{ + if (!isfinite(angle) || angle < 0 || angle > 180) { + return [[FBErrorBuilder.builder withDescription:@"Hinge angle must be a finite number between 0 and 180 degrees"] buildError:error]; + } + if (!self.fb_supportsSimulatedHingeAngle) { + return [[FBErrorBuilder.builder withDescription:@"The device does not report an available hinge"] buildError:error]; + } +#if !TARGET_OS_TV && !TARGET_OS_WATCH + static FBIOCFSerialize serialize; + static FBIOHIDEventCreateVendorDefinedEvent createEvent; + static FBIOHIDEventSystemClientCreate createClient; + static FBIOHIDEventSystemClientDispatchEvent dispatchEvent; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + void *handle = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_LAZY); + if (NULL != handle) { + serialize = (FBIOCFSerialize)dlsym(handle, "IOCFSerialize"); + createEvent = (FBIOHIDEventCreateVendorDefinedEvent)dlsym(handle, "IOHIDEventCreateVendorDefinedEvent"); + createClient = (FBIOHIDEventSystemClientCreate)dlsym(handle, "IOHIDEventSystemClientCreate"); + dispatchEvent = (FBIOHIDEventSystemClientDispatchEvent)dlsym(handle, "IOHIDEventSystemClientDispatchEvent"); + } + }); + if (NULL == serialize || NULL == createEvent || NULL == createClient || NULL == dispatchEvent) { + return [[FBErrorBuilder.builder withDescription:@"The runtime does not provide the required IOKit HID APIs"] buildError:error]; + } + // Matches Device Hub's hinge-slider-control payload (Xcode 27.1). It is an + // IOCF binary serialization, not an NSPropertyListSerialization binary plist. + NSDictionary *payload = @{ + @"provider": @"com.apple.Virtualization.VirtualMachines", + @"source": @"hinge-slider-control", + @"type": @"range", + @"value": @(angle), + }; + CFDataRef data = serialize((__bridge CFTypeRef)payload, 1); + if (NULL == data) { + return [[FBErrorBuilder.builder withDescription:@"Cannot serialize the simulated hinge event"] buildError:error]; + } + CFTypeRef event = createEvent(kCFAllocatorDefault, mach_absolute_time(), 0xff61, 0x5b, 0, + (uint8_t *)CFDataGetBytePtr(data), CFDataGetLength(data), 0); + CFRelease(data); + CFTypeRef client = createClient(kCFAllocatorDefault); + BOOL canDispatch = NULL != event && NULL != client; + if (canDispatch) { + dispatchEvent(client, event); + } + if (NULL != event) { + CFRelease(event); + } + if (NULL != client) { + CFRelease(client); + } + return canDispatch || [[FBErrorBuilder.builder withDescription:@"Cannot create the simulated hinge HID event or client"] buildError:error]; +#else + return NO; +#endif +} + +@end diff --git a/WebDriverAgentLib/Commands/FBCustomCommands.m b/WebDriverAgentLib/Commands/FBCustomCommands.m index 08cf756376..8c6c8a7753 100644 --- a/WebDriverAgentLib/Commands/FBCustomCommands.m +++ b/WebDriverAgentLib/Commands/FBCustomCommands.m @@ -29,6 +29,7 @@ #import "XCUIApplication.h" #import "XCUIApplication+FBHelpers.h" #import "XCUIDevice+FBHelpers.h" +#import "XCUIDevice+FBHinge.h" #import "XCUIDevice+FBVoiceOver.h" #import "XCUIElement.h" #import "XCUIElement+FBIsVisible.h" diff --git a/WebDriverAgentLib/WebDriverAgentLib.h b/WebDriverAgentLib/WebDriverAgentLib.h index de916ee2e1..230daa8639 100644 --- a/WebDriverAgentLib/WebDriverAgentLib.h +++ b/WebDriverAgentLib/WebDriverAgentLib.h @@ -47,6 +47,7 @@ FOUNDATION_EXPORT const unsigned char WebDriverAgentLib_VersionString[]; #import #import #import +#import #import #import #import diff --git a/WebDriverAgentTests/UnitTests/FBCustomCommandsTests.m b/WebDriverAgentTests/UnitTests/FBCustomCommandsTests.m index ff9a8b19ca..4292af1118 100644 --- a/WebDriverAgentTests/UnitTests/FBCustomCommandsTests.m +++ b/WebDriverAgentTests/UnitTests/FBCustomCommandsTests.m @@ -14,7 +14,7 @@ #import "FBSession.h" #import "FBResponsePayload.h" #import "RouteResponse.h" -#import "XCUIDevice+FBHelpers.h" +#import "XCUIDevice+FBHinge.h" #import "Doubles/XCUIElementDouble.h" #if !TARGET_OS_TV && __clang_major__ >= 15 From b3202158545b7c5bc45e030ab036b8be9ebdb6e5 Mon Sep 17 00:00:00 2001 From: Kazuaki Matsuo Date: Sat, 3 Oct 2026 01:52:53 -0700 Subject: [PATCH 12/29] refactor: separate hinge capabilities and delegate screen lookup --- .../Categories/XCUIDevice+FBHinge.h | 9 +- .../Categories/XCUIDevice+FBHinge.m | 96 ++++++++------ WebDriverAgentLib/Commands/FBCustomCommands.m | 8 +- WebDriverAgentLib/Utilities/FBScreen.m | 45 +++++-- .../IntegrationTests/FBScreenTests.m | 117 +++++++++++++++++- .../UnitTests/FBCustomCommandsTests.m | 4 +- docs/multiple-displays.md | 25 ++-- 7 files changed, 234 insertions(+), 70 deletions(-) diff --git a/WebDriverAgentLib/Categories/XCUIDevice+FBHinge.h b/WebDriverAgentLib/Categories/XCUIDevice+FBHinge.h index 7cc5e5d9bb..12cf70fd70 100644 --- a/WebDriverAgentLib/Categories/XCUIDevice+FBHinge.h +++ b/WebDriverAgentLib/Categories/XCUIDevice+FBHinge.h @@ -12,11 +12,14 @@ NS_ASSUME_NONNULL_BEGIN @interface XCUIDevice (FBHinge) +/** Whether a hinge and the CoreMotion angle-reading APIs are available. */ +- (BOOL)fb_supportsHingeAngleReading; + /** - Whether the runtime reports an available hinge, permitting an injection attempt. - This does not guarantee that the device accepts simulated hinge events. + Whether a hinge and the IOKit APIs needed to attempt injection are available. + This does not guarantee event acceptance, particularly on physical devices. */ -- (BOOL)fb_supportsSimulatedHingeAngle; +- (BOOL)fb_canAttemptSimulatedHingeAngleInjection; /** Reads the current hinge angle in degrees, including changes made outside WDA. diff --git a/WebDriverAgentLib/Categories/XCUIDevice+FBHinge.m b/WebDriverAgentLib/Categories/XCUIDevice+FBHinge.m index 35881e6eda..536ef58e5d 100644 --- a/WebDriverAgentLib/Categories/XCUIDevice+FBHinge.m +++ b/WebDriverAgentLib/Categories/XCUIDevice+FBHinge.m @@ -35,6 +35,7 @@ - (float)angleDegrees; @protocol FBAngleManager + (instancetype)new; + (BOOL)isAvailable; ++ (BOOL)instancesRespondToSelector:(SEL)selector; - (void)startAngleUpdatesToQueue:(NSOperationQueue *)queue handler:(void (^)(id))handler; - (void)stopAngleUpdates; @end @@ -50,20 +51,58 @@ - (void)stopAngleUpdates; }); return angleManagerClass; } + +static BOOL FBHasAvailableHinge(void) +{ + Class managerClass = FBAngleManagerClass(); + return [managerClass respondsToSelector:@selector(isAvailable)] && [managerClass isAvailable]; +} + +typedef struct { + FBIOCFSerialize serialize; + FBIOHIDEventCreateVendorDefinedEvent createEvent; + FBIOHIDEventSystemClientCreate createClient; + FBIOHIDEventSystemClientDispatchEvent dispatchEvent; +} FBHingeInjectionAPI; + +static const FBHingeInjectionAPI *FBHingeInjectionFunctions(void) +{ + static FBHingeInjectionAPI api; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + void *handle = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_LAZY); + if (NULL != handle) { + api.serialize = (FBIOCFSerialize)dlsym(handle, "IOCFSerialize"); + api.createEvent = (FBIOHIDEventCreateVendorDefinedEvent)dlsym(handle, "IOHIDEventCreateVendorDefinedEvent"); + api.createClient = (FBIOHIDEventSystemClientCreate)dlsym(handle, "IOHIDEventSystemClientCreate"); + api.dispatchEvent = (FBIOHIDEventSystemClientDispatchEvent)dlsym(handle, "IOHIDEventSystemClientDispatchEvent"); + } + }); + return NULL != api.serialize && NULL != api.createEvent + && NULL != api.createClient && NULL != api.dispatchEvent ? &api : NULL; +} #endif @implementation XCUIDevice (FBHinge) -- (BOOL)fb_supportsSimulatedHingeAngle +- (BOOL)fb_supportsHingeAngleReading { #if !TARGET_OS_TV && !TARGET_OS_WATCH - Class angleManagerClass = FBAngleManagerClass(); - if (![angleManagerClass respondsToSelector:@selector(isAvailable)]) { - return NO; - } - // Hinge availability permits an injection attempt; it does not guarantee - // that the device accepts the vendor event. Physical devices are unverified. - return [angleManagerClass isAvailable]; + Class managerClass = FBAngleManagerClass(); + return FBHasAvailableHinge() + && [managerClass instancesRespondToSelector:@selector(startAngleUpdatesToQueue:handler:)] + && [managerClass instancesRespondToSelector:@selector(stopAngleUpdates)]; +#else + return NO; +#endif +} + +- (BOOL)fb_canAttemptSimulatedHingeAngleInjection +{ +#if !TARGET_OS_TV && !TARGET_OS_WATCH + // Sensor presence and local APIs permit an attempt, not proof that the + // device's receiver accepts the vendor event. Physical devices are unverified. + return FBHasAvailableHinge() && NULL != FBHingeInjectionFunctions(); #else return NO; #endif @@ -71,17 +110,12 @@ - (BOOL)fb_supportsSimulatedHingeAngle - (nullable NSNumber *)fb_getSimulatedHingeAngle:(NSError **)error { - if (!self.fb_supportsSimulatedHingeAngle) { - [[FBErrorBuilder.builder withDescription:@"The device does not report an available hinge"] buildError:error]; + if (!self.fb_supportsHingeAngleReading) { + [[FBErrorBuilder.builder withDescription:@"Hinge angle reading is unavailable on this device"] buildError:error]; return nil; } #if !TARGET_OS_TV && !TARGET_OS_WATCH id manager = [FBAngleManagerClass() new]; - if (![manager respondsToSelector:@selector(startAngleUpdatesToQueue:handler:)] - || ![manager respondsToSelector:@selector(stopAngleUpdates)]) { - [[FBErrorBuilder.builder withDescription:@"The runtime does not provide the required hinge angle reading APIs"] buildError:error]; - return nil; - } NSOperationQueue *queue = [NSOperationQueue new]; queue.maxConcurrentOperationCount = 1; NSCondition *condition = [NSCondition new]; @@ -130,27 +164,11 @@ - (BOOL)fb_setSimulatedHingeAngle:(double)angle error:(NSError **)error if (!isfinite(angle) || angle < 0 || angle > 180) { return [[FBErrorBuilder.builder withDescription:@"Hinge angle must be a finite number between 0 and 180 degrees"] buildError:error]; } - if (!self.fb_supportsSimulatedHingeAngle) { - return [[FBErrorBuilder.builder withDescription:@"The device does not report an available hinge"] buildError:error]; + if (!self.fb_canAttemptSimulatedHingeAngleInjection) { + return [[FBErrorBuilder.builder withDescription:@"Hinge angle injection requires an available hinge and the IOKit HID APIs"] buildError:error]; } #if !TARGET_OS_TV && !TARGET_OS_WATCH - static FBIOCFSerialize serialize; - static FBIOHIDEventCreateVendorDefinedEvent createEvent; - static FBIOHIDEventSystemClientCreate createClient; - static FBIOHIDEventSystemClientDispatchEvent dispatchEvent; - static dispatch_once_t onceToken; - dispatch_once(&onceToken, ^{ - void *handle = dlopen("/System/Library/Frameworks/IOKit.framework/IOKit", RTLD_LAZY); - if (NULL != handle) { - serialize = (FBIOCFSerialize)dlsym(handle, "IOCFSerialize"); - createEvent = (FBIOHIDEventCreateVendorDefinedEvent)dlsym(handle, "IOHIDEventCreateVendorDefinedEvent"); - createClient = (FBIOHIDEventSystemClientCreate)dlsym(handle, "IOHIDEventSystemClientCreate"); - dispatchEvent = (FBIOHIDEventSystemClientDispatchEvent)dlsym(handle, "IOHIDEventSystemClientDispatchEvent"); - } - }); - if (NULL == serialize || NULL == createEvent || NULL == createClient || NULL == dispatchEvent) { - return [[FBErrorBuilder.builder withDescription:@"The runtime does not provide the required IOKit HID APIs"] buildError:error]; - } + const FBHingeInjectionAPI *api = FBHingeInjectionFunctions(); // Matches Device Hub's hinge-slider-control payload (Xcode 27.1). It is an // IOCF binary serialization, not an NSPropertyListSerialization binary plist. NSDictionary *payload = @{ @@ -159,17 +177,17 @@ - (BOOL)fb_setSimulatedHingeAngle:(double)angle error:(NSError **)error @"type": @"range", @"value": @(angle), }; - CFDataRef data = serialize((__bridge CFTypeRef)payload, 1); + CFDataRef data = api->serialize((__bridge CFTypeRef)payload, 1); if (NULL == data) { return [[FBErrorBuilder.builder withDescription:@"Cannot serialize the simulated hinge event"] buildError:error]; } - CFTypeRef event = createEvent(kCFAllocatorDefault, mach_absolute_time(), 0xff61, 0x5b, 0, - (uint8_t *)CFDataGetBytePtr(data), CFDataGetLength(data), 0); + CFTypeRef event = api->createEvent(kCFAllocatorDefault, mach_absolute_time(), 0xff61, 0x5b, 0, + (uint8_t *)CFDataGetBytePtr(data), CFDataGetLength(data), 0); CFRelease(data); - CFTypeRef client = createClient(kCFAllocatorDefault); + CFTypeRef client = api->createClient(kCFAllocatorDefault); BOOL canDispatch = NULL != event && NULL != client; if (canDispatch) { - dispatchEvent(client, event); + api->dispatchEvent(client, event); } if (NULL != event) { CFRelease(event); diff --git a/WebDriverAgentLib/Commands/FBCustomCommands.m b/WebDriverAgentLib/Commands/FBCustomCommands.m index 8c6c8a7753..338f97bc44 100644 --- a/WebDriverAgentLib/Commands/FBCustomCommands.m +++ b/WebDriverAgentLib/Commands/FBCustomCommands.m @@ -784,9 +784,9 @@ + (NSString *)timeZone + (id)handleGetSimulatedHingeAngle:(FBRouteRequest *)request { - if (!XCUIDevice.sharedDevice.fb_supportsSimulatedHingeAngle) { + if (!XCUIDevice.sharedDevice.fb_supportsHingeAngleReading) { return FBResponseWithStatus([FBCommandStatus unsupportedOperationErrorWithMessage: - @"The device does not report an available hinge" traceback:nil]); + @"Hinge angle reading is unavailable on this device" traceback:nil]); } NSError *error; NSNumber *angle = [XCUIDevice.sharedDevice fb_getSimulatedHingeAngle:&error]; @@ -802,9 +802,9 @@ + (NSString *)timeZone return FBResponseWithStatus([FBCommandStatus invalidArgumentErrorWithMessage: @"'angle' must be a finite number between 0 and 180 degrees" traceback:nil]); } - if (!XCUIDevice.sharedDevice.fb_supportsSimulatedHingeAngle) { + if (!XCUIDevice.sharedDevice.fb_canAttemptSimulatedHingeAngleInjection) { return FBResponseWithStatus([FBCommandStatus unsupportedOperationErrorWithMessage: - @"The device does not report an available hinge" traceback:nil]); + @"Hinge angle injection requires an available hinge and the IOKit HID APIs" traceback:nil]); } NSError *error; if (![XCUIDevice.sharedDevice fb_setSimulatedHingeAngle:[angle doubleValue] error:&error]) { diff --git a/WebDriverAgentLib/Utilities/FBScreen.m b/WebDriverAgentLib/Utilities/FBScreen.m index 16e2b36588..e212514973 100644 --- a/WebDriverAgentLib/Utilities/FBScreen.m +++ b/WebDriverAgentLib/Utilities/FBScreen.m @@ -42,27 +42,54 @@ @implementation FBScreen return result.copy; } -+ (nullable XCUIScreen *)screenWithDisplayID:(long long)displayID error:(NSError **)error +// Kept separate from sharedDevice so legacy lookup behavior can be tested. ++ (nullable XCUIScreen *)screenWithDisplayID:(long long)displayID + device:(XCUIDevice *)device + error:(NSError **)error { - NSArray *screens = [XCUIDevice.sharedDevice screensOrError:error]; + BOOL hasNativeLookup = [device respondsToSelector:@selector(screenWithDisplayID:orError:)]; + BOOL needsEnumeratedLookup = !hasNativeLookup; + NSError *lookupError = nil; + if (hasNativeLookup) { + XCUIScreen *screen = [device screenWithDisplayID:displayID orError:&lookupError]; + if (nil != screen && !CGRectIsEmpty(screen.bounds)) { + return screen; + } + // XCTest may return a zero-sized placeholder for an inactive wireless + // display which screensOrError: omits. Confirm its availability by listing. + needsEnumeratedLookup = nil != screen; + } + // Enumerate only for legacy/placeholder lookups or to enrich a lookup error. + NSError *enumerationError = nil; + NSArray *screens = [device screensOrError:&enumerationError]; if (nil == screens) { + if (NULL != error) { + *error = lookupError ?: enumerationError; + } return nil; } + NSMutableArray *availableIDs = [NSMutableArray arrayWithCapacity:screens.count]; for (XCUIScreen *screen in screens) { - if (screen.displayID == displayID) { + if (needsEnumeratedLookup && screen.displayID == displayID) { return screen; } - } - NSMutableArray *availableIDs = [NSMutableArray arrayWithCapacity:screens.count]; - for (XCUIScreen *screen in screens) { [availableIDs addObject:@(screen.displayID)]; } - [[FBErrorBuilder.builder withDescriptionFormat:@"No display with id %lld is available. Available display ids: [%@]", - displayID, [availableIDs componentsJoinedByString:@", "]] - buildError:error]; + FBErrorBuilder *builder = [FBErrorBuilder.builder withDescriptionFormat: + @"No display with id %lld is available. Available display ids: [%@]", + displayID, [availableIDs componentsJoinedByString:@", "]]; + if (nil != lookupError) { + [builder withInnerError:lookupError]; + } + [builder buildError:error]; return nil; } ++ (nullable XCUIScreen *)screenWithDisplayID:(long long)displayID error:(NSError **)error +{ + return [self screenWithDisplayID:displayID device:XCUIDevice.sharedDevice error:error]; +} + + (nullable XCUIScreen *)currentScreenWithError:(NSError **)error { NSNumber *displayID = FBConfiguration.sharedInstance.currentDisplayId; diff --git a/WebDriverAgentTests/IntegrationTests/FBScreenTests.m b/WebDriverAgentTests/IntegrationTests/FBScreenTests.m index e2cd6cc48c..e21a1bd168 100644 --- a/WebDriverAgentTests/IntegrationTests/FBScreenTests.m +++ b/WebDriverAgentTests/IntegrationTests/FBScreenTests.m @@ -17,6 +17,48 @@ #import "FBScreenshot.h" #import "XCUIScreen.h" +@interface FBScreen (FBLookupTesting) ++ (nullable XCUIScreen *)screenWithDisplayID:(long long)displayID device:(id)device error:(NSError **)error; +@end + +@interface FBEmptyScreenDouble : NSObject +@property (nonatomic) CGRect bounds; +@end +@implementation FBEmptyScreenDouble +@end + +@interface FBScreenDeviceDouble : NSObject +@property (nonatomic) BOOL nativeLookupAvailable; +@property (nonatomic) NSUInteger enumerationCount; +@property (nonatomic, strong) XCUIScreen *nativeScreen; +@property (nonatomic, strong) NSArray *screens; +@property (nonatomic, strong) NSError *lookupError; +@property (nonatomic, strong) NSError *enumerationError; +@end + +@implementation FBScreenDeviceDouble +- (BOOL)respondsToSelector:(SEL)selector +{ + return selector == @selector(screenWithDisplayID:orError:) + ? self.nativeLookupAvailable : [super respondsToSelector:selector]; +} +- (XCUIScreen *)screenWithDisplayID:(long long)displayID orError:(NSError **)error +{ + if (NULL != error) { + *error = self.lookupError; + } + return self.nativeScreen; +} +- (NSArray *)screensOrError:(NSError **)error +{ + self.enumerationCount++; + if (NULL != error) { + *error = self.enumerationError; + } + return self.screens; +} +@end + @interface FBScreenTests : FBIntegrationTestCase @end @@ -65,9 +107,13 @@ - (void)testScreens - (void)testScreenWithDisplayID { NSError *error = nil; - XCUIScreen *screen = [FBScreen screenWithDisplayID:[FBScreen displayID] error:&error]; - XCTAssertNil(error); - XCTAssertEqual(screen.displayID, [FBScreen displayID]); + for (NSDictionary *entry in [FBScreen screensWithError:&error]) { + long long displayID = [entry[@"displayId"] longLongValue]; + XCUIScreen *screen = [FBScreen screenWithDisplayID:displayID error:&error]; + XCTAssertNotNil(screen); + XCTAssertNil(error); + XCTAssertEqual(screen.displayID, displayID); + } XCTAssertNil([FBScreen screenWithDisplayID:[self unknownDisplayID] error:&error]); XCTAssertNotNil(error); @@ -80,6 +126,71 @@ - (void)testScreenWithDisplayID XCTAssertTrue([error.localizedDescription containsString:availableDisplays]); } +- (void)testNativeScreenLookupDoesNotEnumerateOnSuccess +{ + FBScreenDeviceDouble *device = [FBScreenDeviceDouble new]; + device.nativeLookupAvailable = YES; + device.nativeScreen = XCUIScreen.mainScreen; + NSError *error = nil; + XCTAssertEqualObjects([FBScreen screenWithDisplayID:device.nativeScreen.displayID device:device error:&error], + device.nativeScreen); + XCTAssertNil(error); + XCTAssertEqual(device.enumerationCount, 0UL); +} + +- (void)testNativeScreenPlaceholderMustBeAvailable +{ + FBScreenDeviceDouble *device = [FBScreenDeviceDouble new]; + device.nativeLookupAvailable = YES; + device.nativeScreen = (id)[FBEmptyScreenDouble new]; + device.screens = @[XCUIScreen.mainScreen]; + NSError *error = nil; + XCTAssertNil([FBScreen screenWithDisplayID:-1 device:device error:&error]); + XCTAssertNotNil(error); + XCTAssertEqual(device.enumerationCount, 1UL); +} + +- (void)testNativeScreenLookupFailureIncludesAvailableIDs +{ + FBScreenDeviceDouble *device = [FBScreenDeviceDouble new]; + device.nativeLookupAvailable = YES; + device.screens = @[XCUIScreen.mainScreen]; + device.lookupError = [NSError errorWithDomain:@"NativeLookup" code:1 userInfo:nil]; + NSError *error = nil; + XCTAssertNil([FBScreen screenWithDisplayID:-1 device:device error:&error]); + XCTAssertEqual(device.enumerationCount, 1UL); + XCTAssertTrue([error.localizedDescription containsString:@"Available display ids"]); + XCTAssertEqualObjects(error.userInfo[NSUnderlyingErrorKey], device.lookupError); +} + +- (void)testNativeScreenLookupErrorSurvivesEnumerationFailure +{ + FBScreenDeviceDouble *device = [FBScreenDeviceDouble new]; + device.nativeLookupAvailable = YES; + device.lookupError = [NSError errorWithDomain:@"NativeLookup" code:1 userInfo:nil]; + device.enumerationError = [NSError errorWithDomain:@"Enumeration" code:2 userInfo:nil]; + NSError *error = nil; + XCTAssertNil([FBScreen screenWithDisplayID:-1 device:device error:&error]); + XCTAssertEqualObjects(error, device.lookupError); +} + +- (void)testScreenLookupWithoutNativeSelector +{ + FBScreenDeviceDouble *device = [FBScreenDeviceDouble new]; + device.screens = @[XCUIScreen.mainScreen]; + NSError *error = nil; + XCTAssertEqualObjects([FBScreen screenWithDisplayID:XCUIScreen.mainScreen.displayID device:device error:&error], + XCUIScreen.mainScreen); + XCTAssertNil(error); + XCTAssertEqual(device.enumerationCount, 1UL); + XCTAssertNil([FBScreen screenWithDisplayID:-1 device:device error:&error]); + XCTAssertTrue([error.localizedDescription containsString:@"Available display ids"]); + device.screens = nil; + device.enumerationError = [NSError errorWithDomain:@"Enumeration" code:2 userInfo:nil]; + XCTAssertNil([FBScreen screenWithDisplayID:-1 device:device error:&error]); + XCTAssertEqualObjects(error, device.enumerationError); +} + - (void)testCurrentScreenDefaultsToMainScreen { NSError *error = nil; diff --git a/WebDriverAgentTests/UnitTests/FBCustomCommandsTests.m b/WebDriverAgentTests/UnitTests/FBCustomCommandsTests.m index 4292af1118..29da75061d 100644 --- a/WebDriverAgentTests/UnitTests/FBCustomCommandsTests.m +++ b/WebDriverAgentTests/UnitTests/FBCustomCommandsTests.m @@ -97,7 +97,7 @@ - (void)testHingeAngleReading { NSError *error = nil; NSNumber *angle = [XCUIDevice.sharedDevice fb_getSimulatedHingeAngle:&error]; - BOOL available = XCUIDevice.sharedDevice.fb_supportsSimulatedHingeAngle; + BOOL available = XCUIDevice.sharedDevice.fb_supportsHingeAngleReading; if (available) { XCTAssertNotNil(angle); XCTAssertNil(error); @@ -132,7 +132,7 @@ - (void)testHingeAngleRejectsInvalidArguments - (void)testHingeAngleRejectsUnsupportedDevices { - XCTSkipIf(XCUIDevice.sharedDevice.fb_supportsSimulatedHingeAngle, @"Requires a device without an available hinge"); + XCTSkipIf(XCUIDevice.sharedDevice.fb_canAttemptSimulatedHingeAngleInjection, @"Requires a device without an available hinge"); for (NSNumber *angle in @[@0, @90, @180]) { XCTAssertEqualObjects([self hingeResponseWithArguments:@{@"angle": angle}][@"value"][@"error"], @"unsupported operation"); diff --git a/docs/multiple-displays.md b/docs/multiple-displays.md index 44e5a97da4..6975585955 100644 --- a/docs/multiple-displays.md +++ b/docs/multiple-displays.md @@ -29,22 +29,27 @@ On an iPhone Duo simulator, `POST /session/:sessionId/wda/device/hingeAngle` wit `{"angle": 90}` sends a hinge event without operating Device Hub's UI. The angle must be a finite number from `0` (closed) to `180` (fully open); fractional values are supported. This uses a private HID protocol verified with Xcode 27.1 on the -iPhone Duo simulator. WDA checks runtime hinge availability instead of a fixed -model identifier and permits injection attempts on physical devices as well. -Physical-device behavior is unverified. Devices without an available hinge, -tvOS, and watchOS return `unsupported operation`. - -The response confirms event dispatch, not that the device applied the angle. Wait for the expected app layout before -continuing, since folding completes asynchronously. The command does not rotate +iPhone Duo simulator. POST checks for a hinge and the required IOKit serialization, +event creation, and dispatch APIs. This permits an injection attempt; it does not +establish that the device accepts this vendor payload. Physical devices may attempt +injection when those prerequisites exist, but event acceptance and the resulting +angle/layout changes on a physical Duo remain unverified. Devices without these +prerequisites, tvOS, and watchOS return `unsupported operation`. + +The response confirms event dispatch, not that the device applied the angle. +Wait for the expected app layout before continuing, since folding completes +asynchronously. The command does not rotate the device or update `currentDisplayId`; select the desired display separately. `GET /session/:sessionId/wda/device/hingeAngle` reads the current angle in degrees and returns a numeric value, for example `{"value": 90.5}`. It reads CoreMotion rather than remembering the last angle sent by WDA, so changes made outside WDA are reflected too. The request waits up to five seconds for a valid reading and -returns an error if none arrives. Devices without an available hinge return -`unsupported operation`. A reading during a fold may reflect an intermediate -angle; it does not wait for the transition to finish. +returns an error if none arrives. GET separately checks for a hinge and the +CoreMotion subscription APIs; it does not depend on the IOKit injection APIs. +Devices without these reading prerequisites return `unsupported operation`. +A reading during a fold may reflect an intermediate angle; it does not wait for +the transition to finish. Both hinge angle endpoints are also available without a session: `GET /wda/device/hingeAngle` and `POST /wda/device/hingeAngle`. They operate on From 91baed9bb744b472d687f59dc013a106f53671a3 Mon Sep 17 00:00:00 2001 From: Kazuaki Matsuo Date: Sat, 3 Oct 2026 08:41:02 -0700 Subject: [PATCH 13/29] fix: guard display APIs for older XCTest versions --- .../Utilities/FBActiveAppDetectionPoint.m | 2 +- WebDriverAgentLib/Utilities/FBScreen.m | 26 +++++++++++++++---- .../IntegrationTests/FBScreenTests.m | 17 ++++++++++++ 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/WebDriverAgentLib/Utilities/FBActiveAppDetectionPoint.m b/WebDriverAgentLib/Utilities/FBActiveAppDetectionPoint.m index 534f5020af..788fd1406c 100644 --- a/WebDriverAgentLib/Utilities/FBActiveAppDetectionPoint.m +++ b/WebDriverAgentLib/Utilities/FBActiveAppDetectionPoint.m @@ -61,7 +61,7 @@ + (instancetype)sharedInstance // display. On Duo this can select the outer SpringBoard while the app is // foreground on the inner display. Let callers use their active-app fallback // instead of treating an element from a different display as authoritative. - if (!screen.isMainScreen) { + if (screen.displayID != XCUIScreen.mainScreen.displayID) { return nil; } } diff --git a/WebDriverAgentLib/Utilities/FBScreen.m b/WebDriverAgentLib/Utilities/FBScreen.m index e212514973..877dfbc2bc 100644 --- a/WebDriverAgentLib/Utilities/FBScreen.m +++ b/WebDriverAgentLib/Utilities/FBScreen.m @@ -13,12 +13,26 @@ #import "FBXCodeCompatibility.h" #import "XCUIDevice.h" #import "XCUIScreen.h" +#import + +static NSArray *FBScreensForDevice(id device, NSError **error) +{ + SEL selector = NSSelectorFromString(@"screensOrError:"); + if ([device respondsToSelector:selector]) { + return ((NSArray *(*)(id, SEL, NSError **))objc_msgSend)(device, selector, error); + } + // Older XCTest versions expose screens on XCUIScreen instead of XCUIDevice. + if ([XCUIScreen respondsToSelector:NSSelectorFromString(@"screens")]) { + return XCUIScreen.screens; + } + return @[XCUIScreen.mainScreen]; +} @implementation FBScreen + (nullable NSArray *> *)screensWithError:(NSError **)error { - NSArray *screens = [XCUIDevice.sharedDevice screensOrError:error]; + NSArray *screens = FBScreensForDevice(XCUIDevice.sharedDevice, error); if (nil == screens) { return nil; } @@ -47,12 +61,14 @@ + (nullable XCUIScreen *)screenWithDisplayID:(long long)displayID device:(XCUIDevice *)device error:(NSError **)error { - BOOL hasNativeLookup = [device respondsToSelector:@selector(screenWithDisplayID:orError:)]; + SEL lookupSelector = NSSelectorFromString(@"screenWithDisplayID:orError:"); + BOOL hasNativeLookup = [device respondsToSelector:lookupSelector]; BOOL needsEnumeratedLookup = !hasNativeLookup; NSError *lookupError = nil; if (hasNativeLookup) { - XCUIScreen *screen = [device screenWithDisplayID:displayID orError:&lookupError]; - if (nil != screen && !CGRectIsEmpty(screen.bounds)) { + XCUIScreen *screen = ((XCUIScreen *(*)(id, SEL, long long, NSError **))objc_msgSend)( + device, lookupSelector, displayID, &lookupError); + if ([screen respondsToSelector:NSSelectorFromString(@"bounds")] && !CGRectIsEmpty(screen.bounds)) { return screen; } // XCTest may return a zero-sized placeholder for an inactive wireless @@ -61,7 +77,7 @@ + (nullable XCUIScreen *)screenWithDisplayID:(long long)displayID } // Enumerate only for legacy/placeholder lookups or to enrich a lookup error. NSError *enumerationError = nil; - NSArray *screens = [device screensOrError:&enumerationError]; + NSArray *screens = FBScreensForDevice(device, &enumerationError); if (nil == screens) { if (NULL != error) { *error = lookupError ?: enumerationError; diff --git a/WebDriverAgentTests/IntegrationTests/FBScreenTests.m b/WebDriverAgentTests/IntegrationTests/FBScreenTests.m index e21a1bd168..9adf70a24d 100644 --- a/WebDriverAgentTests/IntegrationTests/FBScreenTests.m +++ b/WebDriverAgentTests/IntegrationTests/FBScreenTests.m @@ -29,6 +29,7 @@ @implementation FBEmptyScreenDouble @interface FBScreenDeviceDouble : NSObject @property (nonatomic) BOOL nativeLookupAvailable; +@property (nonatomic) BOOL enumerationUnavailable; @property (nonatomic) NSUInteger enumerationCount; @property (nonatomic, strong) XCUIScreen *nativeScreen; @property (nonatomic, strong) NSArray *screens; @@ -39,6 +40,9 @@ @interface FBScreenDeviceDouble : NSObject @implementation FBScreenDeviceDouble - (BOOL)respondsToSelector:(SEL)selector { + if (selector == NSSelectorFromString(@"screensOrError:") && self.enumerationUnavailable) { + return NO; + } return selector == @selector(screenWithDisplayID:orError:) ? self.nativeLookupAvailable : [super respondsToSelector:selector]; } @@ -191,6 +195,19 @@ - (void)testScreenLookupWithoutNativeSelector XCTAssertEqualObjects(error, device.enumerationError); } +- (void)testScreenLookupWithoutDeviceScreenSelectors +{ + FBScreenDeviceDouble *device = [FBScreenDeviceDouble new]; + device.enumerationUnavailable = YES; + NSError *error = nil; + XCTAssertEqualObjects([FBScreen screenWithDisplayID:XCUIScreen.mainScreen.displayID device:device error:&error], + XCUIScreen.mainScreen); + XCTAssertNil(error); + XCTAssertNil([FBScreen screenWithDisplayID:-1 device:device error:&error]); + XCTAssertNotNil(error); + XCTAssertEqual(device.enumerationCount, 0UL); +} + - (void)testCurrentScreenDefaultsToMainScreen { NSError *error = nil; From b655bf2dd939c09006257f009c7eebc8f9481186 Mon Sep 17 00:00:00 2001 From: Kazuaki Matsuo Date: Sat, 3 Oct 2026 09:03:08 -0700 Subject: [PATCH 14/29] refactor: name hinge timeout and document runtime loading --- WebDriverAgentLib/Categories/XCUIDevice+FBHinge.m | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/WebDriverAgentLib/Categories/XCUIDevice+FBHinge.m b/WebDriverAgentLib/Categories/XCUIDevice+FBHinge.m index 536ef58e5d..5d37f053cc 100644 --- a/WebDriverAgentLib/Categories/XCUIDevice+FBHinge.m +++ b/WebDriverAgentLib/Categories/XCUIDevice+FBHinge.m @@ -15,6 +15,8 @@ #import "FBErrorBuilder.h" #if !TARGET_OS_TV && !TARGET_OS_WATCH +static const NSTimeInterval FBHingeAngleReadingTimeout = 5.0; + typedef CFDataRef (*FBIOCFSerialize)(CFTypeRef object, CFOptionFlags options); typedef CFTypeRef (*FBIOHIDEventCreateVendorDefinedEvent)(CFAllocatorRef allocator, uint64_t timestamp, @@ -45,6 +47,8 @@ - (void)stopAngleUpdates; static Class angleManagerClass; static dispatch_once_t onceToken; dispatch_once(&onceToken, ^{ + // This path also loads on physical devices (verified on an iPad mini). + // Framework/class presence does not imply that a hinge sensor is available. if (NULL != dlopen("/System/Library/Frameworks/CoreMotion.framework/CoreMotion", RTLD_LAZY)) { angleManagerClass = NSClassFromString(@"CMAngleManager"); } @@ -65,6 +69,8 @@ static BOOL FBHasAvailableHinge(void) FBIOHIDEventSystemClientDispatchEvent dispatchEvent; } FBHingeInjectionAPI; +// Loads and caches the injection APIs; NULL means at least one is unavailable. +// Resolving these functions does not dispatch an event or confirm its acceptance. static const FBHingeInjectionAPI *FBHingeInjectionFunctions(void) { static FBHingeInjectionAPI api; @@ -138,7 +144,7 @@ - (nullable NSNumber *)fb_getSimulatedHingeAngle:(NSError **)error } [condition unlock]; }]; - NSDate *deadline = [NSDate dateWithTimeIntervalSinceNow:5.0]; + NSDate *deadline = [NSDate dateWithTimeIntervalSinceNow:FBHingeAngleReadingTimeout]; [condition lock]; while (nil == reading) { if (![condition waitUntilDate:deadline]) { @@ -151,7 +157,8 @@ - (nullable NSNumber *)fb_getSimulatedHingeAngle:(NSError **)error [manager stopAngleUpdates]; } if (nil == result) { - [[FBErrorBuilder.builder withDescription:@"Timed out after 5 seconds waiting for a valid hinge angle"] buildError:error]; + [[FBErrorBuilder.builder withDescriptionFormat:@"Timed out after %g seconds waiting for a valid hinge angle", + FBHingeAngleReadingTimeout] buildError:error]; } return result; #else From 44ab7ce6abae4edd68a75e968f298a3281ce5d21 Mon Sep 17 00:00:00 2001 From: Kazuaki Matsuo Date: Sat, 3 Oct 2026 09:06:52 -0700 Subject: [PATCH 15/29] docs: explain why hinge events bypass the existing HID helper --- WebDriverAgentLib/Categories/XCUIDevice+FBHinge.m | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/WebDriverAgentLib/Categories/XCUIDevice+FBHinge.m b/WebDriverAgentLib/Categories/XCUIDevice+FBHinge.m index 5d37f053cc..14ec3cf5bc 100644 --- a/WebDriverAgentLib/Categories/XCUIDevice+FBHinge.m +++ b/WebDriverAgentLib/Categories/XCUIDevice+FBHinge.m @@ -175,6 +175,10 @@ - (BOOL)fb_setSimulatedHingeAngle:(double)angle error:(NSError **)error return [[FBErrorBuilder.builder withDescription:@"Hinge angle injection requires an available hinge and the IOKit HID APIs"] buildError:error]; } #if !TARGET_OS_TV && !TARGET_OS_WATCH + // WDA's fb_performIOHIDEventWithPage:usage:duration:error: uses XCDeviceEvent + // with page, usage, and duration only. The XCDeviceEvent interfaces checked + // in Xcode 27.1 provide no way to attach the vendor payload containing the + // hinge angle, so this command creates and dispatches an IOKit event directly. const FBHingeInjectionAPI *api = FBHingeInjectionFunctions(); // Matches Device Hub's hinge-slider-control payload (Xcode 27.1). It is an // IOCF binary serialization, not an NSPropertyListSerialization binary plist. From feeb4d16bbb68b6f9136bcbbd334a340726cd751 Mon Sep 17 00:00:00 2001 From: Kazuaki Matsuo Date: Sat, 3 Oct 2026 09:10:54 -0700 Subject: [PATCH 16/29] revert unnecessary chnages in the readme --- README.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 4992598ecd..a054c652d5 100644 --- a/README.md +++ b/README.md @@ -10,13 +10,12 @@ WebDriverAgent is a [WebDriver server](https://w3c.github.io/webdriver/webdriver-spec.html) implementation for iOS that can be used to remote control iOS devices. It allows you to launch & kill applications, tap & scroll views or confirm view presence on a screen. This makes it a perfect tool for application end-to-end testing or general purpose device automation. It works by linking `XCTest.framework` and calling Apple's API to execute commands directly on a device. WebDriverAgent is developed for end-to-end testing and is successfully adopted by [Appium](http://appium.io) via [XCUITest driver](https://github.com/appium/appium-xcuitest-driver). ## Features - -- Both iOS and tvOS platforms are supported with devices & simulators -- Implements most of [WebDriver Spec](https://w3c.github.io/webdriver/webdriver-spec.html) -- Implements part of [Mobile JSON Wire Protocol Spec](https://github.com/SeleniumHQ/mobile-spec/blob/master/spec-draft.md) -- USB support for devices is implemented via [appium-ios-device](https://github.com/appium/appium-ios-device) library and has zero dependencies on third-party tools. -- Easy development cycle as it can be launched & debugged directly via Xcode -- Use [Mac2Driver](https://github.com/appium/appium-mac2-driver) to automate macOS apps +* Both iOS and tvOS platforms are supported with devices & simulators +* Implements most of [WebDriver Spec](https://w3c.github.io/webdriver/webdriver-spec.html) +* Implements part of [Mobile JSON Wire Protocol Spec](https://github.com/SeleniumHQ/mobile-spec/blob/master/spec-draft.md) +* USB support for devices is implemented via [appium-ios-device](https://github.com/appium/appium-ios-device) library and has zero dependencies on third-party tools. +* Easy development cycle as it can be launched & debugged directly via Xcode +* Use [Mac2Driver](https://github.com/appium/appium-mac2-driver) to automate macOS apps ## Getting Started On This Repository From 4c69596396399610ccf1f8d80c73362bba54ab8d Mon Sep 17 00:00:00 2001 From: Kazuaki Matsuo Date: Sat, 3 Oct 2026 09:25:43 -0700 Subject: [PATCH 17/29] refactor: unify display checks and scroll geometry --- WebDriverAgent.xcodeproj/project.pbxproj | 4 + .../Categories/XCUIElement+FBScrolling.m | 61 +++-- .../Utilities/FBW3CActionsSynthesizer.m | 2 +- .../IntegrationTests/FBDisplayGestureTests.m | 216 ++++++++++++++++++ .../IntegrationTests/FBScreenTests.m | 169 -------------- .../FBW3CTouchActionsIntegrationTests.m | 14 +- docs/multiple-displays.md | 3 + 7 files changed, 277 insertions(+), 192 deletions(-) create mode 100644 WebDriverAgentTests/IntegrationTests/FBDisplayGestureTests.m diff --git a/WebDriverAgent.xcodeproj/project.pbxproj b/WebDriverAgent.xcodeproj/project.pbxproj index 2c9da2fd99..40d11108e3 100644 --- a/WebDriverAgent.xcodeproj/project.pbxproj +++ b/WebDriverAgent.xcodeproj/project.pbxproj @@ -587,6 +587,7 @@ 715AFAC11FFA29180053896D /* FBScreen.h in Headers */ = {isa = PBXBuildFile; fileRef = 715AFABF1FFA29180053896D /* FBScreen.h */; }; 715AFAC21FFA29180053896D /* FBScreen.m in Sources */ = {isa = PBXBuildFile; fileRef = 715AFAC01FFA29180053896D /* FBScreen.m */; }; 715AFAC41FFA2AAF0053896D /* FBScreenTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 715AFAC31FFA2AAF0053896D /* FBScreenTests.m */; }; + 8EB95DF348A9438BBEAB4501 /* FBDisplayGestureTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 8EB95DF348A9438BBEAB4502 /* FBDisplayGestureTests.m */; }; 715D554B2229891B00524509 /* FBExceptionHandlerTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 715D554A2229891B00524509 /* FBExceptionHandlerTests.m */; }; 715D5773224DE02E00DA2D99 /* libxml2.tbd in Frameworks */ = {isa = PBXBuildFile; fileRef = 716C9345224D540C004B8542 /* libxml2.tbd */; }; 715D5774224DE05400DA2D99 /* libxml2.tbd in Frameworks */ = {isa = PBXBuildFile; fileRef = 716C9345224D540C004B8542 /* libxml2.tbd */; }; @@ -1627,6 +1628,7 @@ 715AFABF1FFA29180053896D /* FBScreen.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FBScreen.h; sourceTree = ""; }; 715AFAC01FFA29180053896D /* FBScreen.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = FBScreen.m; sourceTree = ""; }; 715AFAC31FFA2AAF0053896D /* FBScreenTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = FBScreenTests.m; sourceTree = ""; }; + 8EB95DF348A9438BBEAB4502 /* FBDisplayGestureTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = FBDisplayGestureTests.m; sourceTree = ""; }; 715D554A2229891B00524509 /* FBExceptionHandlerTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = FBExceptionHandlerTests.m; sourceTree = ""; }; 71649EC82518C19C0087F212 /* IOSTestSettings.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = IOSTestSettings.xcconfig; sourceTree = ""; }; 716C9342224D53A1004B8542 /* XCTest.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = XCTest.framework; path = Platforms/AppleTVOS.platform/Developer/Library/Frameworks/XCTest.framework; sourceTree = DEVELOPER_DIR; }; @@ -2704,6 +2706,7 @@ 7119E1EB1E891F8600D0B125 /* FBPickerWheelSelectTests.m */, 71ACF5B7242F2FDC00F0AAD4 /* FBSafariAlertTests.m */, 715AFAC31FFA2AAF0053896D /* FBScreenTests.m */, + 8EB95DF348A9438BBEAB4502 /* FBDisplayGestureTests.m */, EE55B3261D1D54CF003AAAEC /* FBScrollingTests.m */, 7152EB2F1F41F9960047EEFF /* FBSessionIntegrationTests.m */, EE26409A1D0EB5E8009BE6B0 /* FBTapTest.m */, @@ -4861,6 +4864,7 @@ 719CD8FF2126C90200C7D0C2 /* FBAutoAlertsHandlerTests.m in Sources */, EE2202131ECC612200A29571 /* FBIntegrationTestCase.m in Sources */, 715AFAC41FFA2AAF0053896D /* FBScreenTests.m in Sources */, + 8EB95DF348A9438BBEAB4501 /* FBDisplayGestureTests.m in Sources */, 71BB58EC2B96328700CB9BFE /* FBScreenRecordingRequest.m in Sources */, EE22021E1ECC618900A29571 /* FBTapTest.m in Sources */, 71930C472066434000D3AFEC /* FBPasteboardTests.m in Sources */, diff --git a/WebDriverAgentLib/Categories/XCUIElement+FBScrolling.m b/WebDriverAgentLib/Categories/XCUIElement+FBScrolling.m index 8a90b1f133..61b1d06b6d 100644 --- a/WebDriverAgentLib/Categories/XCUIElement+FBScrolling.m +++ b/WebDriverAgentLib/Categories/XCUIElement+FBScrolling.m @@ -9,6 +9,7 @@ #import "XCUIElement+FBScrolling.h" #import "FBErrorBuilder.h" +#import "FBExceptions.h" #import "FBLogger.h" #import "FBScreen.h" #import "FBMacros.h" @@ -37,16 +38,30 @@ #if !TARGET_OS_TV -static CGRect FBScrollFrameInApplication(CGRect frame, XCUIApplication *application) +static XCUIScreen *FBScreenForScrolling(void) { - XCUIScreen *screen = [FBScreen currentScreenWithError:nil]; - if (nil != screen && !screen.isMainScreen) { + NSError *error = nil; + XCUIScreen *screen = [FBScreen currentScreenWithError:&error]; + if (nil == screen) { + // Match active-app detection: never continue on a different coordinate space + // when the selected display is unavailable, including for void scroll APIs. + @throw [NSException exceptionWithName:FBInvalidArgumentException + reason:error.localizedDescription + userInfo:nil]; + } + return screen; +} + +static CGRect FBScrollFrameInApplication(CGRect frame, XCUIScreen *screen, + UIInterfaceOrientation orientation) +{ + if (screen.displayID != XCUIScreen.mainScreen.displayID) { // On secondary displays, XCTest's visibleFrame is in portrait screen // coordinates, whereas wdFrame uses the application's current orientation. // Rotate the clipped visible rect before normalizing it against the anchor. CGSize size = CGSizeMake(CGRectGetWidth(screen.bounds) / screen.scale, CGRectGetHeight(screen.bounds) / screen.scale); - frame = FBRectFromPortraitCoordinates(frame, size, application.interfaceOrientation); + frame = FBRectFromPortraitCoordinates(frame, size, orientation); } return frame; } @@ -58,7 +73,7 @@ - (BOOL)fb_scrollDownByNormalizedDistance:(CGFloat)distance anchorElement:(XCUIE - (BOOL)fb_scrollLeftByNormalizedDistance:(CGFloat)distance anchorElement:(XCUIElement *)anchorElement; - (BOOL)fb_scrollRightByNormalizedDistance:(CGFloat)distance anchorElement:(XCUIElement *)anchorElement; - (BOOL)fb_scrollByNormalizedVector:(CGVector)normalizedScrollVector anchorElement:(XCUIElement *)anchorElement; -- (BOOL)fb_scrollByVector:(CGVector)vector anchorElement:(XCUIElement *)anchorElement error:(NSError **)error; +- (BOOL)fb_scrollByVector:(CGVector)vector inFrame:(CGRect)frame anchorElement:(XCUIElement *)anchorElement error:(NSError **)error; @end @@ -142,6 +157,7 @@ - (BOOL)fb_scrollToVisibleWithNormalizedScrollDistance:(CGFloat)normalizedScroll scrollDirection:(FBXCUIElementScrollDirection)scrollDirection error:(NSError **)error { + FBScreenForScrolling(); FBXCElementSnapshotWrapper *prescrollSnapshot = [FBXCElementSnapshotWrapper ensureWrapped:[self fb_customSnapshot]]; if (prescrollSnapshot.isWDVisible) { @@ -276,14 +292,18 @@ - (BOOL)fb_scrollToVisibleWithNormalizedScrollDistance:(CGFloat)normalizedScroll // Trying fb_cachedSnapshot first FBXCElementSnapshotWrapper *targetCellSnapshotWrapped = [FBXCElementSnapshotWrapper ensureWrapped:[self fb_customSnapshot]]; targetCellSnapshot = [targetCellSnapshotWrapped fb_parentCellSnapshot]; + XCUIScreen *screen = FBScreenForScrolling(); + UIInterfaceOrientation orientation = self.application.interfaceOrientation; CGRect visibleFrame = FBScrollFrameInApplication( - [FBXCElementSnapshotWrapper ensureWrapped:targetCellSnapshot].fb_visibleFrame, self.application); + [FBXCElementSnapshotWrapper ensureWrapped:targetCellSnapshot].fb_visibleFrame, screen, orientation); CGVector scrollVector = CGVectorMake(visibleFrame.size.width - targetCellSnapshot.frame.size.width, visibleFrame.size.height - targetCellSnapshot.frame.size.height ); scrollViewWrapped = [FBXCElementSnapshotWrapper ensureWrapped:[scrollViewElement fb_customSnapshot]]; + CGRect scrollingFrame = FBScrollFrameInApplication(scrollViewWrapped.visibleFrame, screen, orientation); return [scrollViewWrapped fb_scrollByVector:scrollVector + inFrame:scrollingFrame anchorElement:scrollViewElement error:error]; } @@ -315,7 +335,8 @@ @implementation FBXCElementSnapshotWrapper (FBScrolling) - (CGRect)scrollingFrameWithAnchor:(XCUIElement *)anchorElement { - return FBScrollFrameInApplication(self.visibleFrame, anchorElement.application); + XCUIScreen *screen = FBScreenForScrolling(); + return FBScrollFrameInApplication(self.visibleFrame, screen, anchorElement.application.interfaceOrientation); } - (BOOL)fb_scrollUpByNormalizedDistance:(CGFloat)distance @@ -349,32 +370,36 @@ - (BOOL)fb_scrollByNormalizedVector:(CGVector)normalizedScrollVector CGVector scrollVector = CGVectorMake(CGRectGetWidth(frame) * normalizedScrollVector.dx, CGRectGetHeight(frame) * normalizedScrollVector.dy ); - return [self fb_scrollByVector:scrollVector anchorElement:anchorElement error:nil]; + return [self fb_scrollByVector:scrollVector inFrame:frame anchorElement:anchorElement error:nil]; } - (BOOL)fb_scrollByVector:(CGVector)vector + inFrame:(CGRect)frame anchorElement:(XCUIElement *)anchorElement error:(NSError **)error { - CGRect frame = [self scrollingFrameWithAnchor:anchorElement]; - CGVector scrollBoundingVector = CGVectorMake( - CGRectGetWidth(frame) * FBScrollTouchProportion, - CGRectGetHeight(frame) * FBScrollTouchProportion - ); - scrollBoundingVector.dx = (CGFloat)floor(copysign(scrollBoundingVector.dx, vector.dx)); - scrollBoundingVector.dy = (CGFloat)floor(copysign(scrollBoundingVector.dy, vector.dy)); - NSInteger preciseScrollAttemptsCount = 20; CGVector CGZeroVector = CGVectorMake(0, 0); BOOL shouldFinishScrolling = NO; while (!shouldFinishScrolling) { + CGVector scrollBoundingVector = CGVectorMake( + (CGFloat)floor(copysign(CGRectGetWidth(frame) * FBScrollTouchProportion, vector.dx)), + (CGFloat)floor(copysign(CGRectGetHeight(frame) * FBScrollTouchProportion, vector.dy))); CGVector scrollVector = CGVectorMake(fabs(vector.dx) > fabs(scrollBoundingVector.dx) ? scrollBoundingVector.dx : vector.dx, fabs(vector.dy) > fabs(scrollBoundingVector.dy) ? scrollBoundingVector.dy : vector.dy); vector = CGVectorMake(vector.dx - scrollVector.dx, vector.dy - scrollVector.dy); shouldFinishScrolling = FBVectorFuzzyEqualToVector(vector, CGZeroVector, 1) || --preciseScrollAttemptsCount <= 0; - if (![self fb_scrollAncestorScrollViewByVectorWithinScrollViewFrame:scrollVector anchorElement:anchorElement error:error]){ + if (![self fb_scrollAncestorScrollViewByVectorWithinScrollViewFrame:scrollVector + inFrame:frame + anchorElement:anchorElement + error:error]) { return NO; } + if (!shouldFinishScrolling) { + // Share geometry within a drag, but allow the next drag to observe rotation + // or display changes that occurred while the previous drag was executing. + frame = [self scrollingFrameWithAnchor:anchorElement]; + } } return YES; } @@ -389,10 +414,10 @@ - (CGVector)fb_normalizedHitPointOffsetForScrollingVector:(CGVector)scrollingVec } - (BOOL)fb_scrollAncestorScrollViewByVectorWithinScrollViewFrame:(CGVector)vector + inFrame:(CGRect)scrollingFrame anchorElement:(XCUIElement *)anchorElement error:(NSError **)error { - CGRect scrollingFrame = [self scrollingFrameWithAnchor:anchorElement]; // wdFrame matches scrollingFrame's coordinate space; raw .frame can be pre-scaled or // dimension-swapped and drift out of sync with it (appium/appium#16185). CGRect anchorFrame = anchorElement.wdFrame; diff --git a/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m b/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m index 2ff1214836..10dba33ead 100644 --- a/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m +++ b/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m @@ -878,7 +878,7 @@ - (nullable XCSynthesizedEventRecord *)synthesizeWithError:(NSError **)error } XCSynthesizedEventRecord *eventRecord; self.displayCorrection = CGVectorMake(0, 0); - if (screen.isMainScreen) { + if (screen.displayID == XCUIScreen.mainScreen.displayID) { eventRecord = [[XCSynthesizedEventRecord alloc] initWithName:@"W3C Touch Action" interfaceOrientation:self.application.interfaceOrientation]; } else { diff --git a/WebDriverAgentTests/IntegrationTests/FBDisplayGestureTests.m b/WebDriverAgentTests/IntegrationTests/FBDisplayGestureTests.m new file mode 100644 index 0000000000..d78e70e3d3 --- /dev/null +++ b/WebDriverAgentTests/IntegrationTests/FBDisplayGestureTests.m @@ -0,0 +1,216 @@ +/** + * Copyright (c) 2015-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#import + +#import "FBIntegrationTestCase.h" +#import "FBConfiguration.h" +#import "FBScreen.h" +#import "FBExceptions.h" + +#import "XCUIApplication+FBTouchAction.h" +#import "XCUIElement+FBScrolling.h" + +@interface FBDisplayGestureTests : FBIntegrationTestCase +@property (nonatomic) NSNumber *previousDisplayId; +@property (nonatomic) XCUIElement *canvas; +@end + +@implementation FBDisplayGestureTests + +- (void)setUp +{ + [super setUp]; + self.previousDisplayId = FBConfiguration.sharedInstance.currentDisplayId; + [self launchApplication]; + [self.testedApplication.buttons[@"coordinate-probe"] tap]; + self.canvas = self.testedApplication.otherElements[@"coordinate-canvas"]; + + // The fixture reports the screen it actually occupies. Do not assume the main + // screen is active: Duo changes displays when its hinge changes position. + NSArray *size = [self measurement][@"screenSize"]; + NSNumber *displayId = nil; + for (NSDictionary *screen in [FBScreen screensWithError:nil]) { + CGFloat width = [screen[@"bounds"][@"width"] doubleValue] / [screen[@"scale"] doubleValue]; + CGFloat height = [screen[@"bounds"][@"height"] doubleValue] / [screen[@"scale"] doubleValue]; + if (fabs(MAX(width, height) - MAX(size[0].doubleValue, size[1].doubleValue)) < 1 + && fabs(MIN(width, height) - MIN(size[0].doubleValue, size[1].doubleValue)) < 1) { + displayId = screen[@"displayId"]; + break; + } + } + XCTAssertNotNil(displayId, @"Cannot identify the fixture's display"); + FBConfiguration.sharedInstance.currentDisplayId = displayId; +} + +- (void)tearDown +{ + FBConfiguration.sharedInstance.currentDisplayId = self.previousDisplayId; + [super tearDown]; +} + +- (NSDictionary *)measurement +{ + NSString *value = self.testedApplication.staticTexts[@"probe-status"].label; + NSError *error = nil; + NSDictionary *result = [NSJSONSerialization JSONObjectWithData:[value dataUsingEncoding:NSUTF8StringEncoding] + options:0 error:&error]; + XCTAssertNil(error); + return result; +} + +- (void)performMoves:(NSArray *)moves expectedEnd:(CGPoint)expected +{ + NSUInteger count = [[self measurement][@"count"] unsignedIntegerValue]; + NSMutableArray *items = [NSMutableArray arrayWithArray:@[ + moves.firstObject, + @{@"type": @"pointerDown", @"button": @0}, + @{@"type": @"pause", @"duration": @100}, + ]]; + if (moves.count > 1) { + [items addObjectsFromArray:[moves subarrayWithRange:NSMakeRange(1, moves.count - 1)]]; + } + [items addObject:@{@"type": @"pointerUp", @"button": @0}]; + NSError *error = nil; + NSArray *actions = @[@{ + @"type": @"pointer", @"id": @"finger", @"parameters": @{@"pointerType": @"touch"}, @"actions": items, + }]; + XCTAssertTrue([self.testedApplication fb_performW3CActions:actions elementCache:nil error:&error], @"%@", error); + NSDictionary *result = [self measurement]; + XCTAssertEqual([result[@"count"] unsignedIntegerValue], count + 1); + XCTAssertEqualObjects(result[@"phase"], @"ended"); + NSArray *actual = result[@"last"]; + XCTAssertEqualWithAccuracy(actual[0].doubleValue, expected.x, 1); + XCTAssertEqualWithAccuracy(actual[1].doubleValue, expected.y, 1); +} + +- (NSDictionary *)moveFrom:(id)origin x:(CGFloat)x y:(CGFloat)y duration:(NSUInteger)duration +{ + return @{@"type": @"pointerMove", @"origin": origin, @"x": @(x), @"y": @(y), @"duration": @(duration)}; +} + +- (void)testViewportAndElementCorners +{ + NSDictionary *geometry = [self measurement]; + NSArray *bounds = geometry[@"canvasBounds"]; + NSArray *frame = geometry[@"canvasWindowRect"]; + CGFloat width = bounds[0].doubleValue, height = bounds[1].doubleValue; + for (NSNumber *x in @[@40, @(width - 40)]) { + for (NSNumber *y in @[@40, @(height - 40)]) { + CGPoint point = CGPointMake(x.doubleValue, y.doubleValue); + [self performMoves:@[[self moveFrom:@"viewport" + x:frame[0].doubleValue + point.x y:frame[1].doubleValue + point.y duration:0]] + expectedEnd:point]; + [self performMoves:@[[self moveFrom:self.canvas x:point.x - width / 2 y:point.y - height / 2 duration:0]] + expectedEnd:point]; + } + } +} + +- (void)testElementHitpoint +{ + NSArray *bounds = [self measurement][@"canvasBounds"]; + [self performMoves:@[@{@"type": @"pointerMove", @"origin": self.canvas, @"duration": @0}] + expectedEnd:CGPointMake(bounds[0].doubleValue / 2, bounds[1].doubleValue / 2)]; +} + +- (void)testPointerRelativeMovesPreserveOriginSpace +{ + NSDictionary *geometry = [self measurement]; + NSArray *bounds = geometry[@"canvasBounds"]; + NSArray *frame = geometry[@"canvasWindowRect"]; + CGPoint center = CGPointMake(bounds[0].doubleValue / 2, bounds[1].doubleValue / 2); + for (NSDictionary *start in @[ + [self moveFrom:self.canvas x:0 y:0 duration:0], + [self moveFrom:@"viewport" x:frame[0].doubleValue + center.x y:frame[1].doubleValue + center.y duration:0], + ]) { + [self performMoves:@[start, + [self moveFrom:@"pointer" x:0 y:-40 duration:200], + [self moveFrom:@"pointer" x:0 y:-40 duration:200]] + expectedEnd:CGPointMake(center.x, center.y - 80)]; + } +} + +- (void)testMixedOriginsWithinOneTouch +{ + NSDictionary *geometry = [self measurement]; + NSArray *bounds = geometry[@"canvasBounds"]; + NSArray *frame = geometry[@"canvasWindowRect"]; + CGPoint center = CGPointMake(bounds[0].doubleValue / 2, bounds[1].doubleValue / 2); + for (NSNumber *startAtElement in @[@YES, @NO]) { + NSDictionary *start = startAtElement.boolValue + ? [self moveFrom:self.canvas x:0 y:0 duration:0] + : [self moveFrom:@"viewport" x:frame[0].doubleValue + center.x y:frame[1].doubleValue + center.y duration:0]; + NSDictionary *next = startAtElement.boolValue + ? [self moveFrom:@"viewport" x:frame[0].doubleValue + center.x y:frame[1].doubleValue + center.y - 40 duration:200] + : [self moveFrom:self.canvas x:0 y:-40 duration:200]; + [self performMoves:@[start, next, [self moveFrom:@"pointer" x:0 y:-40 duration:200]] + expectedEnd:CGPointMake(center.x, center.y - 80)]; + } +} + +- (void)testScrollRemainsInForegroundAndMovesContent +{ + [self.testedApplication.buttons[@"Scroll"] tap]; + XCUIElement *table = self.testedApplication.tables[@"probe-table"]; + CGFloat before = [[self measurement][@"scrollY"] doubleValue]; + // Exceeds the per-drag limit, exercising geometry refresh between drags. + [table fb_scrollDownByNormalizedDistance:1.0]; + XCTAssertEqual(self.testedApplication.state, XCUIApplicationStateRunningForeground); + CGFloat after = [[self measurement][@"scrollY"] doubleValue]; + XCTAssertGreaterThan(after, before + 10); + [table fb_scrollUpByNormalizedDistance:0.5]; + XCTAssertLessThan([[self measurement][@"scrollY"] doubleValue], after - 10); +} + +- (void)testDirectionalScrollRejectsUnavailableDisplay +{ + [self.testedApplication.buttons[@"Scroll"] tap]; + XCUIElement *table = self.testedApplication.tables[@"probe-table"]; + CGFloat before = [[self measurement][@"scrollY"] doubleValue]; + NSNumber *displayId = FBConfiguration.sharedInstance.currentDisplayId; + @try { + FBConfiguration.sharedInstance.currentDisplayId = @(-1); + XCTAssertThrowsSpecificNamed([table fb_scrollDownByNormalizedDistance:0.5], NSException, FBInvalidArgumentException); + XCTAssertThrowsSpecificNamed([table fb_scrollUpByNormalizedDistance:0.5], NSException, FBInvalidArgumentException); + XCTAssertThrowsSpecificNamed([table fb_scrollLeftByNormalizedDistance:0.5], NSException, FBInvalidArgumentException); + XCTAssertThrowsSpecificNamed([table fb_scrollRightByNormalizedDistance:0.5], NSException, FBInvalidArgumentException); + } @finally { + FBConfiguration.sharedInstance.currentDisplayId = displayId; + } + XCTAssertEqualWithAccuracy([[self measurement][@"scrollY"] doubleValue], before, 1); +} + +- (void)testScrollToElementRejectsUnavailableDisplay +{ + [self.testedApplication.buttons[@"Scroll"] tap]; + XCUIElement *cell = self.testedApplication.cells[@"probe-row-13"]; + NSNumber *displayId = FBConfiguration.sharedInstance.currentDisplayId; + @try { + FBConfiguration.sharedInstance.currentDisplayId = @(-1); + NSError *error = nil; + XCTAssertThrowsSpecificNamed([cell fb_scrollToVisibleWithError:&error], NSException, FBInvalidArgumentException); + } @finally { + FBConfiguration.sharedInstance.currentDisplayId = displayId; + } +} + +- (void)testScrollToElementKeepsTheWholeCellVisible +{ + [self.testedApplication.buttons[@"Scroll"] tap]; + XCUIElement *cell = self.testedApplication.cells[@"probe-row-13"]; + NSError *error = nil; + XCTAssertTrue([cell fb_scrollToVisibleWithError:&error], @"%@", error); + XCTAssertEqual(self.testedApplication.state, XCUIApplicationStateRunningForeground); + NSArray *frame = [self measurement][@"canvasWindowRect"]; + XCTAssertTrue(cell.hittable); + XCTAssertGreaterThanOrEqual(CGRectGetMinY(cell.frame), frame[1].doubleValue - 1); + XCTAssertLessThanOrEqual(CGRectGetMaxY(cell.frame), frame[1].doubleValue + frame[3].doubleValue + 1); +} + +@end diff --git a/WebDriverAgentTests/IntegrationTests/FBScreenTests.m b/WebDriverAgentTests/IntegrationTests/FBScreenTests.m index a28c59a46c..9adf70a24d 100644 --- a/WebDriverAgentTests/IntegrationTests/FBScreenTests.m +++ b/WebDriverAgentTests/IntegrationTests/FBScreenTests.m @@ -16,8 +16,6 @@ #import "FBScreenRecordingRequest.h" #import "FBScreenshot.h" #import "XCUIScreen.h" -#import "XCUIApplication+FBTouchAction.h" -#import "XCUIElement+FBScrolling.h" @interface FBScreen (FBLookupTesting) + (nullable XCUIScreen *)screenWithDisplayID:(long long)displayID device:(id)device error:(NSError **)error; @@ -302,170 +300,3 @@ - (void)testScreenScale } @end - - -@interface FBDisplayGestureTests : FBIntegrationTestCase -@property (nonatomic) NSNumber *previousDisplayId; -@property (nonatomic) XCUIElement *canvas; -@end - -@implementation FBDisplayGestureTests - -- (void)setUp -{ - [super setUp]; - self.previousDisplayId = FBConfiguration.sharedInstance.currentDisplayId; - [self launchApplication]; - [self.testedApplication.buttons[@"coordinate-probe"] tap]; - self.canvas = self.testedApplication.otherElements[@"coordinate-canvas"]; - - // The fixture reports the screen it actually occupies. Do not assume the main - // screen is active: Duo changes displays when its hinge changes position. - NSArray *size = [self measurement][@"screenSize"]; - NSNumber *displayId = nil; - for (NSDictionary *screen in [FBScreen screensWithError:nil]) { - CGFloat width = [screen[@"bounds"][@"width"] doubleValue] / [screen[@"scale"] doubleValue]; - CGFloat height = [screen[@"bounds"][@"height"] doubleValue] / [screen[@"scale"] doubleValue]; - if (fabs(MAX(width, height) - MAX(size[0].doubleValue, size[1].doubleValue)) < 1 - && fabs(MIN(width, height) - MIN(size[0].doubleValue, size[1].doubleValue)) < 1) { - displayId = screen[@"displayId"]; - break; - } - } - XCTAssertNotNil(displayId, @"Cannot identify the fixture's display"); - FBConfiguration.sharedInstance.currentDisplayId = displayId; -} - -- (void)tearDown -{ - FBConfiguration.sharedInstance.currentDisplayId = self.previousDisplayId; - [super tearDown]; -} - -- (NSDictionary *)measurement -{ - NSString *value = self.testedApplication.staticTexts[@"probe-status"].label; - NSError *error = nil; - NSDictionary *result = [NSJSONSerialization JSONObjectWithData:[value dataUsingEncoding:NSUTF8StringEncoding] - options:0 error:&error]; - XCTAssertNil(error); - return result; -} - -- (void)performMoves:(NSArray *)moves expectedEnd:(CGPoint)expected -{ - NSUInteger count = [[self measurement][@"count"] unsignedIntegerValue]; - NSMutableArray *items = [NSMutableArray arrayWithArray:@[ - moves.firstObject, - @{@"type": @"pointerDown", @"button": @0}, - @{@"type": @"pause", @"duration": @100}, - ]]; - if (moves.count > 1) { - [items addObjectsFromArray:[moves subarrayWithRange:NSMakeRange(1, moves.count - 1)]]; - } - [items addObject:@{@"type": @"pointerUp", @"button": @0}]; - NSError *error = nil; - NSArray *actions = @[@{ - @"type": @"pointer", @"id": @"finger", @"parameters": @{@"pointerType": @"touch"}, @"actions": items, - }]; - XCTAssertTrue([self.testedApplication fb_performW3CActions:actions elementCache:nil error:&error], @"%@", error); - NSDictionary *result = [self measurement]; - XCTAssertEqual([result[@"count"] unsignedIntegerValue], count + 1); - XCTAssertEqualObjects(result[@"phase"], @"ended"); - NSArray *actual = result[@"last"]; - XCTAssertEqualWithAccuracy(actual[0].doubleValue, expected.x, 1); - XCTAssertEqualWithAccuracy(actual[1].doubleValue, expected.y, 1); -} - -- (NSDictionary *)moveFrom:(id)origin x:(CGFloat)x y:(CGFloat)y duration:(NSUInteger)duration -{ - return @{@"type": @"pointerMove", @"origin": origin, @"x": @(x), @"y": @(y), @"duration": @(duration)}; -} - -- (void)testViewportAndElementCorners -{ - NSDictionary *geometry = [self measurement]; - NSArray *bounds = geometry[@"canvasBounds"]; - NSArray *frame = geometry[@"canvasWindowRect"]; - CGFloat width = bounds[0].doubleValue, height = bounds[1].doubleValue; - for (NSNumber *x in @[@40, @(width - 40)]) { - for (NSNumber *y in @[@40, @(height - 40)]) { - CGPoint point = CGPointMake(x.doubleValue, y.doubleValue); - [self performMoves:@[[self moveFrom:@"viewport" - x:frame[0].doubleValue + point.x y:frame[1].doubleValue + point.y duration:0]] - expectedEnd:point]; - [self performMoves:@[[self moveFrom:self.canvas x:point.x - width / 2 y:point.y - height / 2 duration:0]] - expectedEnd:point]; - } - } -} - -- (void)testElementHitpoint -{ - NSArray *bounds = [self measurement][@"canvasBounds"]; - [self performMoves:@[@{@"type": @"pointerMove", @"origin": self.canvas, @"duration": @0}] - expectedEnd:CGPointMake(bounds[0].doubleValue / 2, bounds[1].doubleValue / 2)]; -} - -- (void)testPointerRelativeMovesPreserveOriginSpace -{ - NSDictionary *geometry = [self measurement]; - NSArray *bounds = geometry[@"canvasBounds"]; - NSArray *frame = geometry[@"canvasWindowRect"]; - CGPoint center = CGPointMake(bounds[0].doubleValue / 2, bounds[1].doubleValue / 2); - for (NSDictionary *start in @[ - [self moveFrom:self.canvas x:0 y:0 duration:0], - [self moveFrom:@"viewport" x:frame[0].doubleValue + center.x y:frame[1].doubleValue + center.y duration:0], - ]) { - [self performMoves:@[start, - [self moveFrom:@"pointer" x:0 y:-40 duration:200], - [self moveFrom:@"pointer" x:0 y:-40 duration:200]] - expectedEnd:CGPointMake(center.x, center.y - 80)]; - } -} - -- (void)testMixedOriginsWithinOneTouch -{ - NSDictionary *geometry = [self measurement]; - NSArray *bounds = geometry[@"canvasBounds"]; - NSArray *frame = geometry[@"canvasWindowRect"]; - CGPoint center = CGPointMake(bounds[0].doubleValue / 2, bounds[1].doubleValue / 2); - for (NSNumber *startAtElement in @[@YES, @NO]) { - NSDictionary *start = startAtElement.boolValue - ? [self moveFrom:self.canvas x:0 y:0 duration:0] - : [self moveFrom:@"viewport" x:frame[0].doubleValue + center.x y:frame[1].doubleValue + center.y duration:0]; - NSDictionary *next = startAtElement.boolValue - ? [self moveFrom:@"viewport" x:frame[0].doubleValue + center.x y:frame[1].doubleValue + center.y - 40 duration:200] - : [self moveFrom:self.canvas x:0 y:-40 duration:200]; - [self performMoves:@[start, next, [self moveFrom:@"pointer" x:0 y:-40 duration:200]] - expectedEnd:CGPointMake(center.x, center.y - 80)]; - } -} - -- (void)testScrollRemainsInForegroundAndMovesContent -{ - [self.testedApplication.buttons[@"Scroll"] tap]; - XCUIElement *table = self.testedApplication.tables[@"probe-table"]; - CGFloat before = [[self measurement][@"scrollY"] doubleValue]; - [table fb_scrollDownByNormalizedDistance:0.5]; - XCTAssertEqual(self.testedApplication.state, XCUIApplicationStateRunningForeground); - CGFloat after = [[self measurement][@"scrollY"] doubleValue]; - XCTAssertGreaterThan(after, before + 10); - [table fb_scrollUpByNormalizedDistance:0.5]; - XCTAssertLessThan([[self measurement][@"scrollY"] doubleValue], after - 10); -} - -- (void)testScrollToElementKeepsTheWholeCellVisible -{ - [self.testedApplication.buttons[@"Scroll"] tap]; - XCUIElement *cell = self.testedApplication.cells[@"probe-row-13"]; - NSError *error = nil; - XCTAssertTrue([cell fb_scrollToVisibleWithError:&error], @"%@", error); - XCTAssertEqual(self.testedApplication.state, XCUIApplicationStateRunningForeground); - NSArray *frame = [self measurement][@"canvasWindowRect"]; - XCTAssertTrue(cell.hittable); - XCTAssertGreaterThanOrEqual(CGRectGetMinY(cell.frame), frame[1].doubleValue - 1); - XCTAssertLessThanOrEqual(CGRectGetMaxY(cell.frame), frame[1].doubleValue + frame[3].doubleValue + 1); -} - -@end diff --git a/WebDriverAgentTests/IntegrationTests/FBW3CTouchActionsIntegrationTests.m b/WebDriverAgentTests/IntegrationTests/FBW3CTouchActionsIntegrationTests.m index 57cd13774d..916064db81 100644 --- a/WebDriverAgentTests/IntegrationTests/FBW3CTouchActionsIntegrationTests.m +++ b/WebDriverAgentTests/IntegrationTests/FBW3CTouchActionsIntegrationTests.m @@ -124,16 +124,22 @@ - (void)testUnavailableDisplaySettingIsRejected - (void)testSecondaryDisplaySetting { XCUIScreen *secondary = nil; - for (XCUIScreen *screen in [XCUIDevice.sharedDevice screensOrError:nil]) { - if (!screen.isMainScreen) { - secondary = screen; + NSError *error = nil; + NSArray *> *screens = [FBScreen screensWithError:&error]; + XCTAssertNotNil(screens); + XCTAssertNil(error); + for (NSDictionary *screen in screens) { + long long displayID = [screen[@"displayId"] longLongValue]; + if (displayID != [FBScreen displayID]) { + secondary = [FBScreen screenWithDisplayID:displayID error:&error]; + XCTAssertNotNil(secondary); + XCTAssertNil(error); break; } } if (nil == secondary) { XCTSkip(@"The device has no secondary display"); } - NSError *error = nil; XCSynthesizedEventRecord *baseline = [self tapRecordWithError:&error]; FBConfiguration.sharedInstance.currentDisplayId = @(secondary.displayID); XCSynthesizedEventRecord *selected = [self tapRecordWithError:&error]; diff --git a/docs/multiple-displays.md b/docs/multiple-displays.md index 47b3c15d6e..769480e528 100644 --- a/docs/multiple-displays.md +++ b/docs/multiple-displays.md @@ -24,6 +24,9 @@ W3C touch actions target the selected display. Viewport, element, and pointer origins can be mixed within one action sequence; element offsets remain relative to the element's center. Element scrolling uses the visible area in the same coordinate space as the element frame, including on a rotated secondary display. +WDA scrolling rejects an unavailable selected display instead of using an +uncorrected frame. Each drag uses one converted scrolling frame; subsequent +drags refresh the display and orientation. This gesture support builds on [#1269](https://github.com/appium/WebDriverAgent/pull/1269). From ecb650bbf02dafdbcdf17cc9267a9df8563a9983 Mon Sep 17 00:00:00 2001 From: Kazuaki Matsuo Date: Tue, 6 Oct 2026 00:22:59 -0700 Subject: [PATCH 18/29] fix: guard scrolling orientation APIs on watchOS (#1294) --- .../Categories/XCUIElement+FBScrolling.m | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/WebDriverAgentLib/Categories/XCUIElement+FBScrolling.m b/WebDriverAgentLib/Categories/XCUIElement+FBScrolling.m index 61b1d06b6d..a712456811 100644 --- a/WebDriverAgentLib/Categories/XCUIElement+FBScrolling.m +++ b/WebDriverAgentLib/Categories/XCUIElement+FBScrolling.m @@ -53,16 +53,18 @@ } static CGRect FBScrollFrameInApplication(CGRect frame, XCUIScreen *screen, - UIInterfaceOrientation orientation) + XCUIApplication *application) { +#if !TARGET_OS_WATCH if (screen.displayID != XCUIScreen.mainScreen.displayID) { // On secondary displays, XCTest's visibleFrame is in portrait screen // coordinates, whereas wdFrame uses the application's current orientation. // Rotate the clipped visible rect before normalizing it against the anchor. CGSize size = CGSizeMake(CGRectGetWidth(screen.bounds) / screen.scale, CGRectGetHeight(screen.bounds) / screen.scale); - frame = FBRectFromPortraitCoordinates(frame, size, orientation); + frame = FBRectFromPortraitCoordinates(frame, size, application.interfaceOrientation); } +#endif return frame; } @@ -293,15 +295,14 @@ - (BOOL)fb_scrollToVisibleWithNormalizedScrollDistance:(CGFloat)normalizedScroll FBXCElementSnapshotWrapper *targetCellSnapshotWrapped = [FBXCElementSnapshotWrapper ensureWrapped:[self fb_customSnapshot]]; targetCellSnapshot = [targetCellSnapshotWrapped fb_parentCellSnapshot]; XCUIScreen *screen = FBScreenForScrolling(); - UIInterfaceOrientation orientation = self.application.interfaceOrientation; CGRect visibleFrame = FBScrollFrameInApplication( - [FBXCElementSnapshotWrapper ensureWrapped:targetCellSnapshot].fb_visibleFrame, screen, orientation); + [FBXCElementSnapshotWrapper ensureWrapped:targetCellSnapshot].fb_visibleFrame, screen, self.application); CGVector scrollVector = CGVectorMake(visibleFrame.size.width - targetCellSnapshot.frame.size.width, visibleFrame.size.height - targetCellSnapshot.frame.size.height ); scrollViewWrapped = [FBXCElementSnapshotWrapper ensureWrapped:[scrollViewElement fb_customSnapshot]]; - CGRect scrollingFrame = FBScrollFrameInApplication(scrollViewWrapped.visibleFrame, screen, orientation); + CGRect scrollingFrame = FBScrollFrameInApplication(scrollViewWrapped.visibleFrame, screen, self.application); return [scrollViewWrapped fb_scrollByVector:scrollVector inFrame:scrollingFrame anchorElement:scrollViewElement @@ -336,7 +337,7 @@ @implementation FBXCElementSnapshotWrapper (FBScrolling) - (CGRect)scrollingFrameWithAnchor:(XCUIElement *)anchorElement { XCUIScreen *screen = FBScreenForScrolling(); - return FBScrollFrameInApplication(self.visibleFrame, screen, anchorElement.application.interfaceOrientation); + return FBScrollFrameInApplication(self.visibleFrame, screen, anchorElement.application); } - (BOOL)fb_scrollUpByNormalizedDistance:(CGFloat)distance From 86ae3a1bd1ea0d9cc97dffd5867182257e0b1d5b Mon Sep 17 00:00:00 2001 From: Kazuaki Matsuo Date: Tue, 6 Oct 2026 08:16:53 -0700 Subject: [PATCH 19/29] fix: preserve display-independent actions and correct landscape scrolling --- .../Categories/XCUIElement+FBScrolling.m | 124 ++++++++++-------- .../Utilities/FBW3CActionsSynthesizer.m | 70 ++++++---- .../IntegrationApp/COORDINATE_PROBE.md | 3 +- .../Classes/FBCoordinateProbeViewController.m | 7 + .../IntegrationTests/FBDisplayGestureTests.m | 86 +++++++++++- .../FBW3CTouchActionsIntegrationTests.m | 20 +++ docs/multiple-displays.md | 13 +- 7 files changed, 233 insertions(+), 90 deletions(-) diff --git a/WebDriverAgentLib/Categories/XCUIElement+FBScrolling.m b/WebDriverAgentLib/Categories/XCUIElement+FBScrolling.m index a712456811..1687934fa2 100644 --- a/WebDriverAgentLib/Categories/XCUIElement+FBScrolling.m +++ b/WebDriverAgentLib/Categories/XCUIElement+FBScrolling.m @@ -56,14 +56,11 @@ static CGRect FBScrollFrameInApplication(CGRect frame, XCUIScreen *screen, XCUIApplication *application) { #if !TARGET_OS_WATCH - if (screen.displayID != XCUIScreen.mainScreen.displayID) { - // On secondary displays, XCTest's visibleFrame is in portrait screen - // coordinates, whereas wdFrame uses the application's current orientation. - // Rotate the clipped visible rect before normalizing it against the anchor. - CGSize size = CGSizeMake(CGRectGetWidth(screen.bounds) / screen.scale, - CGRectGetHeight(screen.bounds) / screen.scale); - frame = FBRectFromPortraitCoordinates(frame, size, application.interfaceOrientation); - } + // XCTest's visibleFrame is in portrait screen coordinates on both main and + // secondary displays. The live anchor's frame uses the app's orientation. + CGSize size = CGSizeMake(CGRectGetWidth(screen.bounds) / screen.scale, + CGRectGetHeight(screen.bounds) / screen.scale); + frame = FBRectFromPortraitCoordinates(frame, size, application.interfaceOrientation); #endif return frame; } @@ -74,8 +71,8 @@ - (BOOL)fb_scrollUpByNormalizedDistance:(CGFloat)distance anchorElement:(XCUIEle - (BOOL)fb_scrollDownByNormalizedDistance:(CGFloat)distance anchorElement:(XCUIElement *)anchorElement; - (BOOL)fb_scrollLeftByNormalizedDistance:(CGFloat)distance anchorElement:(XCUIElement *)anchorElement; - (BOOL)fb_scrollRightByNormalizedDistance:(CGFloat)distance anchorElement:(XCUIElement *)anchorElement; -- (BOOL)fb_scrollByNormalizedVector:(CGVector)normalizedScrollVector anchorElement:(XCUIElement *)anchorElement; -- (BOOL)fb_scrollByVector:(CGVector)vector inFrame:(CGRect)frame anchorElement:(XCUIElement *)anchorElement error:(NSError **)error; +- (BOOL)fb_scrollByNormalizedVector:(CGVector)normalizedScrollVector anchorElement:(XCUIElement *)anchorElement error:(NSError **)error; +- (BOOL)fb_scrollByVector:(CGVector)vector inFrame:(CGRect)frame anchorElement:(XCUIElement *)anchorElement velocity:(CGFloat)velocity error:(NSError **)error; @end @@ -159,7 +156,9 @@ - (BOOL)fb_scrollToVisibleWithNormalizedScrollDistance:(CGFloat)normalizedScroll scrollDirection:(FBXCUIElementScrollDirection)scrollDirection error:(NSError **)error { - FBScreenForScrolling(); + if (nil == [FBScreen currentScreenWithError:error]) { + return NO; + } FBXCElementSnapshotWrapper *prescrollSnapshot = [FBXCElementSnapshotWrapper ensureWrapped:[self fb_customSnapshot]]; if (prescrollSnapshot.isWDVisible) { @@ -236,7 +235,7 @@ - (BOOL)fb_scrollToVisibleWithNormalizedScrollDistance:(CGFloat)normalizedScroll // The scroll view's own identity is stable across scroll steps, so it only needs // to be resolved to a live element once, up front; its frame does not, since it can - // change across scroll steps (rotation, keyboard, dynamic layout). + // change across scroll steps (keyboard or dynamic layout). XCUIElement *scrollViewElement = FBLiveElementForSnapshot(scrollView, self.application); if (nil == scrollViewElement) { return @@ -251,34 +250,27 @@ - (BOOL)fb_scrollToVisibleWithNormalizedScrollDistance:(CGFloat)normalizedScroll // Scrolling till cell is visible and get current value of frames while (![self fb_isEquivalentElementSnapshotVisible:prescrollSnapshot] && scrollCount < maxScrollCount) { BOOL didScroll; + __strong NSError *scrollError = nil; @autoreleasepool { - // Re-snapshotting the scroll view every step keeps its frame from drifting too far - // out of sync with the live anchor element's frame used to resolve touch points. scrollViewWrapped = [FBXCElementSnapshotWrapper ensureWrapped:[scrollViewElement fb_customSnapshot]]; - if (targetCellIndex < visibleCellIndex) { - didScroll = scrollDirection == FBXCUIElementScrollDirectionVertical ? - [scrollViewWrapped fb_scrollUpByNormalizedDistance:normalizedScrollDistance - anchorElement:scrollViewElement] : - [scrollViewWrapped fb_scrollLeftByNormalizedDistance:normalizedScrollDistance - anchorElement:scrollViewElement]; - } - else { - didScroll = scrollDirection == FBXCUIElementScrollDirectionVertical ? - [scrollViewWrapped fb_scrollDownByNormalizedDistance:normalizedScrollDistance - anchorElement:scrollViewElement] : - [scrollViewWrapped fb_scrollRightByNormalizedDistance:normalizedScrollDistance - anchorElement:scrollViewElement]; - } + CGFloat distance = targetCellIndex < visibleCellIndex ? normalizedScrollDistance : -normalizedScrollDistance; + CGVector vector = scrollDirection == FBXCUIElementScrollDirectionVertical + ? CGVectorMake(0, distance) : CGVectorMake(distance, 0); + NSError *stepError = nil; + didScroll = [scrollViewWrapped fb_scrollByNormalizedVector:vector + anchorElement:scrollViewElement + error:&stepError]; + scrollError = stepError; scrollCount++; // Wait for scroll animation [self fb_waitUntilStableWithTimeout:FBConfiguration.sharedInstance.animationCoolOffTimeout]; } // The `error` out-param must not be written from inside the autorelease pool above. if (!didScroll) { - return - [[[FBErrorBuilder builder] - withDescriptionFormat:@"Failed to scroll '%@': its frame is empty", self.description] - buildError:error]; + if (error) { + *error = scrollError; + } + return NO; } } @@ -294,18 +286,39 @@ - (BOOL)fb_scrollToVisibleWithNormalizedScrollDistance:(CGFloat)normalizedScroll // Trying fb_cachedSnapshot first FBXCElementSnapshotWrapper *targetCellSnapshotWrapped = [FBXCElementSnapshotWrapper ensureWrapped:[self fb_customSnapshot]]; targetCellSnapshot = [targetCellSnapshotWrapped fb_parentCellSnapshot]; - XCUIScreen *screen = FBScreenForScrolling(); + XCUIScreen *screen = [FBScreen currentScreenWithError:error]; + if (nil == screen) { + return NO; + } CGRect visibleFrame = FBScrollFrameInApplication( [FBXCElementSnapshotWrapper ensureWrapped:targetCellSnapshot].fb_visibleFrame, screen, self.application); - CGVector scrollVector = CGVectorMake(visibleFrame.size.width - targetCellSnapshot.frame.size.width, - visibleFrame.size.height - targetCellSnapshot.frame.size.height - ); + CGRect targetFrame = targetCellSnapshot.frame; + // A cell can be clipped at the leading edge after a drag. Move back toward + // that edge instead of always scrolling down/right and hiding it further. + CGVector scrollVector = CGVectorMake( + CGRectGetMinX(visibleFrame) > CGRectGetMinX(targetFrame) + ? CGRectGetMinX(visibleFrame) - CGRectGetMinX(targetFrame) + : CGRectGetMaxX(visibleFrame) - CGRectGetMaxX(targetFrame), + CGRectGetMinY(visibleFrame) > CGRectGetMinY(targetFrame) + ? CGRectGetMinY(visibleFrame) - CGRectGetMinY(targetFrame) + : CGRectGetMaxY(visibleFrame) - CGRectGetMaxY(targetFrame)); + // UIKit consumes the start of a drag before recognizing a pan. Include that + // margin so a small clipped edge is actually moved into view. + if (fabs(scrollVector.dx) > 1) { + scrollVector.dx += copysign(FBFuzzyPointThreshold, scrollVector.dx); + } + if (fabs(scrollVector.dy) > 1) { + scrollVector.dy += copysign(FBFuzzyPointThreshold, scrollVector.dy); + } + // Short fast corrections can turn into a flick and overshoot the target. + CGFloat velocity = MIN(FBTouchVelocity, MAX(fabs(scrollVector.dx), fabs(scrollVector.dy)) / FBTouchEventDelay); scrollViewWrapped = [FBXCElementSnapshotWrapper ensureWrapped:[scrollViewElement fb_customSnapshot]]; CGRect scrollingFrame = FBScrollFrameInApplication(scrollViewWrapped.visibleFrame, screen, self.application); return [scrollViewWrapped fb_scrollByVector:scrollVector inFrame:scrollingFrame anchorElement:scrollViewElement + velocity:velocity error:error]; } @@ -334,51 +347,57 @@ - (BOOL)fb_isEquivalentElementSnapshotVisible:(id)snapshot @implementation FBXCElementSnapshotWrapper (FBScrolling) -- (CGRect)scrollingFrameWithAnchor:(XCUIElement *)anchorElement -{ - XCUIScreen *screen = FBScreenForScrolling(); - return FBScrollFrameInApplication(self.visibleFrame, screen, anchorElement.application); -} - - (BOOL)fb_scrollUpByNormalizedDistance:(CGFloat)distance anchorElement:(XCUIElement *)anchorElement { - return [self fb_scrollByNormalizedVector:CGVectorMake(0.0, distance) anchorElement:anchorElement]; + FBScreenForScrolling(); + return [self fb_scrollByNormalizedVector:CGVectorMake(0.0, distance) anchorElement:anchorElement error:nil]; } - (BOOL)fb_scrollDownByNormalizedDistance:(CGFloat)distance anchorElement:(XCUIElement *)anchorElement { - return [self fb_scrollByNormalizedVector:CGVectorMake(0.0, -distance) anchorElement:anchorElement]; + FBScreenForScrolling(); + return [self fb_scrollByNormalizedVector:CGVectorMake(0.0, -distance) anchorElement:anchorElement error:nil]; } - (BOOL)fb_scrollLeftByNormalizedDistance:(CGFloat)distance anchorElement:(XCUIElement *)anchorElement { - return [self fb_scrollByNormalizedVector:CGVectorMake(distance, 0.0) anchorElement:anchorElement]; + FBScreenForScrolling(); + return [self fb_scrollByNormalizedVector:CGVectorMake(distance, 0.0) anchorElement:anchorElement error:nil]; } - (BOOL)fb_scrollRightByNormalizedDistance:(CGFloat)distance anchorElement:(XCUIElement *)anchorElement { - return [self fb_scrollByNormalizedVector:CGVectorMake(-distance, 0.0) anchorElement:anchorElement]; + FBScreenForScrolling(); + return [self fb_scrollByNormalizedVector:CGVectorMake(-distance, 0.0) anchorElement:anchorElement error:nil]; } - (BOOL)fb_scrollByNormalizedVector:(CGVector)normalizedScrollVector anchorElement:(XCUIElement *)anchorElement + error:(NSError **)error { - CGRect frame = [self scrollingFrameWithAnchor:anchorElement]; + XCUIScreen *screen = [FBScreen currentScreenWithError:error]; + if (nil == screen) { + return NO; + } + CGRect frame = FBScrollFrameInApplication(self.visibleFrame, screen, anchorElement.application); CGVector scrollVector = CGVectorMake(CGRectGetWidth(frame) * normalizedScrollVector.dx, CGRectGetHeight(frame) * normalizedScrollVector.dy ); - return [self fb_scrollByVector:scrollVector inFrame:frame anchorElement:anchorElement error:nil]; + return [self fb_scrollByVector:scrollVector inFrame:frame anchorElement:anchorElement velocity:FBTouchVelocity error:error]; } - (BOOL)fb_scrollByVector:(CGVector)vector inFrame:(CGRect)frame anchorElement:(XCUIElement *)anchorElement + velocity:(CGFloat)velocity error:(NSError **)error { + // Keep the vector and clipped frame in the same coordinate space throughout + // this multi-drag gesture. Rotation or folding during a gesture is unsupported. NSInteger preciseScrollAttemptsCount = 20; CGVector CGZeroVector = CGVectorMake(0, 0); BOOL shouldFinishScrolling = NO; @@ -393,14 +412,10 @@ - (BOOL)fb_scrollByVector:(CGVector)vector if (![self fb_scrollAncestorScrollViewByVectorWithinScrollViewFrame:scrollVector inFrame:frame anchorElement:anchorElement + velocity:velocity error:error]) { return NO; } - if (!shouldFinishScrolling) { - // Share geometry within a drag, but allow the next drag to observe rotation - // or display changes that occurred while the previous drag was executing. - frame = [self scrollingFrameWithAnchor:anchorElement]; - } } return YES; } @@ -417,6 +432,7 @@ - (CGVector)fb_normalizedHitPointOffsetForScrollingVector:(CGVector)scrollingVec - (BOOL)fb_scrollAncestorScrollViewByVectorWithinScrollViewFrame:(CGVector)vector inFrame:(CGRect)scrollingFrame anchorElement:(XCUIElement *)anchorElement + velocity:(CGFloat)velocity error:(NSError **)error { // wdFrame matches scrollingFrame's coordinate space; raw .frame can be pre-scaled or @@ -440,7 +456,7 @@ - (BOOL)fb_scrollAncestorScrollViewByVectorWithinScrollViewFrame:(CGVector)vecto [startCoordinate pressForDuration:FBTouchEventDelay thenDragToCoordinate:endCoordinate - withVelocity:FBTouchVelocity + withVelocity:velocity thenHoldForDuration:FBTouchEventDelay]; return YES; } diff --git a/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m b/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m index 10dba33ead..9e89fb4838 100644 --- a/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m +++ b/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m @@ -872,35 +872,7 @@ @implementation FBW3CActionsSynthesizer - (nullable XCSynthesizedEventRecord *)synthesizeWithError:(NSError **)error { - XCUIScreen *screen = [FBScreen currentScreenWithError:error]; - if (nil == screen) { - return nil; - } - XCSynthesizedEventRecord *eventRecord; - self.displayCorrection = CGVectorMake(0, 0); - if (screen.displayID == XCUIScreen.mainScreen.displayID) { - eventRecord = [[XCSynthesizedEventRecord alloc] initWithName:@"W3C Touch Action" - interfaceOrientation:self.application.interfaceOrientation]; - } else { - if (![XCSynthesizedEventRecord instancesRespondToSelector:@selector(initWithName:displayID:interfaceOrientation:)]) { - if (error) { - *error = [[FBErrorBuilder.builder - withDescription:@"Actions on a display other than the main one are not supported by this XCTest version"] build]; - } - return nil; - } - eventRecord = [[XCSynthesizedEventRecord alloc] initWithName:@"W3C Touch Action" - displayID:(unsigned long long)screen.displayID - interfaceOrientation:self.application.interfaceOrientation]; - // Viewport coordinates use the main display's size when rotating screenPoint. - // Element coordinates already use their own display. Apply this correction - // per gesture item so mixed origins and pointer-relative moves stay aligned. - XCUIScreen *mainScreen = XCUIScreen.mainScreen; - self.displayCorrection = FBDisplayCoordinateOffset( - CGSizeMake(mainScreen.bounds.size.width / mainScreen.scale, mainScreen.bounds.size.height / mainScreen.scale), - CGSizeMake(screen.bounds.size.width / screen.scale, screen.bounds.size.height / screen.scale), - eventRecord.interfaceOrientation); - } + BOOL requiresDisplay = NO; NSMutableDictionary *> *actionsMapping = [NSMutableDictionary new]; NSMutableArray *actionIds = [NSMutableArray new]; for (NSDictionary *action in self.actions) { @@ -932,9 +904,49 @@ - (nullable XCSynthesizedEventRecord *)synthesizeWithError:(NSError **)error continue; } + if ([action[FB_KEY_TYPE] isEqual:FB_ACTION_TYPE_POINTER]) { + for (NSDictionary *item in actionItems) { + if (![item[FB_ACTION_ITEM_KEY_TYPE] isEqual:FB_ACTION_ITEM_TYPE_PAUSE]) { + requiresDisplay = YES; + break; + } + } + } + [actionIds addObject:actionId]; [actionsMapping setObject:action forKey:actionId]; } + // Keyboard and pause-only sequences have no display coordinates. A display + // disappearing must not prevent these input sources from being synthesized. + XCUIScreen *screen = requiresDisplay ? [FBScreen currentScreenWithError:error] : XCUIScreen.mainScreen; + if (nil == screen) { + return nil; + } + XCSynthesizedEventRecord *eventRecord; + self.displayCorrection = CGVectorMake(0, 0); + if (screen.displayID == XCUIScreen.mainScreen.displayID) { + eventRecord = [[XCSynthesizedEventRecord alloc] initWithName:@"W3C Touch Action" + interfaceOrientation:self.application.interfaceOrientation]; + } else { + if (![XCSynthesizedEventRecord instancesRespondToSelector:@selector(initWithName:displayID:interfaceOrientation:)]) { + if (error) { + *error = [[FBErrorBuilder.builder + withDescription:@"Actions on a display other than the main one are not supported by this XCTest version"] build]; + } + return nil; + } + eventRecord = [[XCSynthesizedEventRecord alloc] initWithName:@"W3C Touch Action" + displayID:(unsigned long long)screen.displayID + interfaceOrientation:self.application.interfaceOrientation]; + // Viewport coordinates use the main display's size when rotating screenPoint. + // Element coordinates already use their own display. Apply this correction + // per gesture item so mixed origins and pointer-relative moves stay aligned. + XCUIScreen *mainScreen = XCUIScreen.mainScreen; + self.displayCorrection = FBDisplayCoordinateOffset( + CGSizeMake(mainScreen.bounds.size.width / mainScreen.scale, mainScreen.bounds.size.height / mainScreen.scale), + CGSizeMake(screen.bounds.size.width / screen.scale, screen.bounds.size.height / screen.scale), + eventRecord.interfaceOrientation); + } for (NSString *actionId in actionIds.copy) { NSDictionary *actionDescription = [actionsMapping objectForKey:actionId]; NSArray *eventPaths = [self eventPathsWithActionDescription:actionDescription forActionId:actionId error:error]; diff --git a/WebDriverAgentTests/IntegrationApp/COORDINATE_PROBE.md b/WebDriverAgentTests/IntegrationApp/COORDINATE_PROBE.md index 820485fa7a..87a21eff23 100644 --- a/WebDriverAgentTests/IntegrationApp/COORDINATE_PROBE.md +++ b/WebDriverAgentTests/IntegrationApp/COORDINATE_PROBE.md @@ -21,7 +21,8 @@ shows 80 numbered rows, each 44 points high. The `probe-status` label exposes `start`, `last`, `phase`, and `count` after a touch. `start` and `last` are **canvas-local app points**. It also exposes -`canvasBounds`, `canvasWindowRect`, `windowSize`, `screenSize`, and `scrollY`. +`canvasBounds`, `canvasWindowRect`, `windowSize`, `screenSize`, `scrollY`, and +`fullyVisibleRows` (row indices whose full rectangles lie inside the table bounds). Geometry is refreshed when the view lays out, including after rotation or window resizing. Touch-related fields are absent before the first touch. diff --git a/WebDriverAgentTests/IntegrationApp/Classes/FBCoordinateProbeViewController.m b/WebDriverAgentTests/IntegrationApp/Classes/FBCoordinateProbeViewController.m index dc72e69463..f407129ca6 100644 --- a/WebDriverAgentTests/IntegrationApp/Classes/FBCoordinateProbeViewController.m +++ b/WebDriverAgentTests/IntegrationApp/Classes/FBCoordinateProbeViewController.m @@ -139,6 +139,13 @@ - (void)publishMeasurement self.measurement[@"windowSize"] = @[@(window.bounds.size.width), @(window.bounds.size.height)]; self.measurement[@"screenSize"] = @[@(window.screen.bounds.size.width), @(window.screen.bounds.size.height)]; self.measurement[@"scrollY"] = @(self.table.contentOffset.y); + NSMutableArray *fullyVisibleRows = [NSMutableArray new]; + for (NSIndexPath *indexPath in self.table.indexPathsForVisibleRows) { + if (CGRectContainsRect(self.table.bounds, [self.table rectForRowAtIndexPath:indexPath])) { + [fullyVisibleRows addObject:@(indexPath.row)]; + } + } + self.measurement[@"fullyVisibleRows"] = fullyVisibleRows; NSData *data = [NSJSONSerialization dataWithJSONObject:self.measurement options:NSJSONWritingSortedKeys error:nil]; diff --git a/WebDriverAgentTests/IntegrationTests/FBDisplayGestureTests.m b/WebDriverAgentTests/IntegrationTests/FBDisplayGestureTests.m index d78e70e3d3..2328fdbe8a 100644 --- a/WebDriverAgentTests/IntegrationTests/FBDisplayGestureTests.m +++ b/WebDriverAgentTests/IntegrationTests/FBDisplayGestureTests.m @@ -15,6 +15,8 @@ #import "XCUIApplication+FBTouchAction.h" #import "XCUIElement+FBScrolling.h" +#import "XCUIDevice+FBRotation.h" +#import "XCUIDevice+FBHinge.h" @interface FBDisplayGestureTests : FBIntegrationTestCase @property (nonatomic) NSNumber *previousDisplayId; @@ -27,10 +29,18 @@ - (void)setUp { [super setUp]; self.previousDisplayId = FBConfiguration.sharedInstance.currentDisplayId; + if (!XCUIDevice.sharedDevice.fb_canAttemptSimulatedHingeAngleInjection) { + [self resetOrientation]; + } [self launchApplication]; [self.testedApplication.buttons[@"coordinate-probe"] tap]; self.canvas = self.testedApplication.otherElements[@"coordinate-canvas"]; + [self selectFixtureDisplay]; +} + +- (void)selectFixtureDisplay +{ // The fixture reports the screen it actually occupies. Do not assume the main // screen is active: Duo changes displays when its hinge changes position. NSArray *size = [self measurement][@"screenSize"]; @@ -159,7 +169,7 @@ - (void)testScrollRemainsInForegroundAndMovesContent [self.testedApplication.buttons[@"Scroll"] tap]; XCUIElement *table = self.testedApplication.tables[@"probe-table"]; CGFloat before = [[self measurement][@"scrollY"] doubleValue]; - // Exceeds the per-drag limit, exercising geometry refresh between drags. + // Exceeds the per-drag limit, exercising multiple drags in one coordinate space. [table fb_scrollDownByNormalizedDistance:1.0]; XCTAssertEqual(self.testedApplication.state, XCUIApplicationStateRunningForeground); CGFloat after = [[self measurement][@"scrollY"] doubleValue]; @@ -194,7 +204,8 @@ - (void)testScrollToElementRejectsUnavailableDisplay @try { FBConfiguration.sharedInstance.currentDisplayId = @(-1); NSError *error = nil; - XCTAssertThrowsSpecificNamed([cell fb_scrollToVisibleWithError:&error], NSException, FBInvalidArgumentException); + XCTAssertFalse([cell fb_scrollToVisibleWithError:&error]); + XCTAssertNotNil(error); } @finally { FBConfiguration.sharedInstance.currentDisplayId = displayId; } @@ -213,4 +224,75 @@ - (void)testScrollToElementKeepsTheWholeCellVisible XCTAssertLessThanOrEqual(CGRectGetMaxY(cell.frame), frame[1].doubleValue + frame[3].doubleValue + 1); } +- (void)verifyLandscapeScrolling:(UIDeviceOrientation)orientation +{ + XCTSkipIf(XCUIDevice.sharedDevice.fb_canAttemptSimulatedHingeAngleInjection, + @"Foldable simulator orientations are covered by testGesturesAcrossFoldStates"); + UIDeviceOrientation previousOrientation = XCUIDevice.sharedDevice.orientation; + [self addTeardownBlock:^{ + [XCUIDevice.sharedDevice fb_setDeviceInterfaceOrientation:previousOrientation]; + }]; + XCTAssertTrue([[XCUIDevice sharedDevice] fb_setDeviceInterfaceOrientation:orientation]); + [self.testedApplication.buttons[@"Scroll"] tap]; + XCUIElement *table = self.testedApplication.tables[@"probe-table"]; + // Choose a nearby offscreen row: the phone's landscape table is intentionally + // short, and this test checks coordinates rather than the 25-scroll limit. + NSUInteger row = (NSUInteger)ceil(table.frame.size.height / 44) + 2; + XCUIElement *cell = self.testedApplication.cells[[NSString stringWithFormat:@"probe-row-%lu", (unsigned long)row]]; + NSError *error = nil; + XCTAssertTrue([cell fb_scrollToVisibleWithError:&error], @"%@", error); + XCTAssertEqual(self.testedApplication.state, XCUIApplicationStateRunningForeground); + XCTAssertGreaterThan([[self measurement][@"scrollY"] doubleValue], 10); + XCTAssertTrue([[self measurement][@"fullyVisibleRows"] containsObject:@(row)]); + XCTAssertTrue(cell.hittable); + XCTAssertGreaterThanOrEqual(CGRectGetMinY(cell.frame), CGRectGetMinY(table.frame) - 1); + XCTAssertLessThanOrEqual(CGRectGetMaxY(cell.frame), CGRectGetMaxY(table.frame) + 1); +} + +- (void)testScrollToElementInLandscapeLeft +{ + [self verifyLandscapeScrolling:UIDeviceOrientationLandscapeLeft]; +} + +- (void)testScrollToElementInLandscapeRight +{ + [self verifyLandscapeScrolling:UIDeviceOrientationLandscapeRight]; +} + +- (void)testGesturesAcrossFoldStates +{ + XCUIDevice *device = XCUIDevice.sharedDevice; + XCTSkipIf(!device.fb_canAttemptSimulatedHingeAngleInjection, @"Requires a foldable simulator"); + NSError *error = nil; + NSNumber *originalAngle = [device fb_getSimulatedHingeAngle:&error]; + XCTAssertNotNil(originalAngle, @"%@", error); + [self addTeardownBlock:^{ + if (nil != originalAngle) { + [device fb_setSimulatedHingeAngle:originalAngle.doubleValue error:nil]; + } + }]; + for (NSNumber *angle in @[@0, @90, @180, @0]) { + [self.testedApplication terminate]; + XCTAssertTrue([device fb_setSimulatedHingeAngle:angle.doubleValue error:&error], @"%@", error); + [self launchApplication]; + [self.testedApplication.buttons[@"coordinate-probe"] tap]; + self.canvas = self.testedApplication.otherElements[@"coordinate-canvas"]; + [self selectFixtureDisplay]; + NSDictionary *geometry = [self measurement]; + NSArray *bounds = geometry[@"canvasBounds"]; + NSArray *frame = geometry[@"canvasWindowRect"]; + CGPoint center = CGPointMake(bounds[0].doubleValue / 2, bounds[1].doubleValue / 2); + [self performMoves:@[[self moveFrom:@"viewport" x:frame[0].doubleValue + center.x + y:frame[1].doubleValue + center.y duration:0], + [self moveFrom:self.canvas x:0 y:-20 duration:200], + [self moveFrom:@"pointer" x:0 y:-20 duration:200]] + expectedEnd:CGPointMake(center.x, center.y - 40)]; + [self.testedApplication.buttons[@"Scroll"] tap]; + XCUIElement *table = self.testedApplication.tables[@"probe-table"]; + [table fb_scrollDownByNormalizedDistance:1.0]; + XCTAssertGreaterThan([[self measurement][@"scrollY"] doubleValue], 10); + XCTAssertEqual(self.testedApplication.state, XCUIApplicationStateRunningForeground); + } +} + @end diff --git a/WebDriverAgentTests/IntegrationTests/FBW3CTouchActionsIntegrationTests.m b/WebDriverAgentTests/IntegrationTests/FBW3CTouchActionsIntegrationTests.m index 916064db81..020e5773ae 100644 --- a/WebDriverAgentTests/IntegrationTests/FBW3CTouchActionsIntegrationTests.m +++ b/WebDriverAgentTests/IntegrationTests/FBW3CTouchActionsIntegrationTests.m @@ -36,6 +36,26 @@ @interface FBW3CTouchActionsIntegrationTestsPart2 : FBIntegrationTestCase @implementation FBW3CTouchActionsIntegrationTestsPart1 +- (void)testDisplayIndependentActionsAfterSelectedDisplayDisappears +{ + [self launchApplication]; + NSNumber *previous = FBConfiguration.sharedInstance.currentDisplayId; + [self addTeardownBlock:^{ FBConfiguration.sharedInstance.currentDisplayId = previous; }]; + NSArray *sequences = @[ + @{@"id": @"keyboard", @"type": @"key", @"actions": @[@{@"type": @"keyDown", @"value": @"a"}, @{@"type": @"keyUp", @"value": @"a"}]}, + @{@"id": @"finger", @"type": @"pointer", @"parameters": @{@"pointerType": @"touch"}, @"actions": @[@{@"type": @"pause", @"duration": @10}]} + ]; + for (NSDictionary *sequence in sequences) { + for (NSNumber *display in @[@1, @987654]) { + FBConfiguration.sharedInstance.currentDisplayId = display; + NSError *error = nil; + FBW3CActionsSynthesizer *synthesizer = [[FBW3CActionsSynthesizer alloc] initWithActions:@[sequence] forApplication:self.testedApplication elementCache:nil error:&error]; + XCSynthesizedEventRecord *record = [synthesizer synthesizeWithError:&error]; + XCTAssertNotNil(record, @"Display-independent %@ should succeed: %@", sequence[@"type"], error); + } + } +} + - (void)verifyGesture:(NSArray *> *)gesture orientation:(UIDeviceOrientation)orientation { [[XCUIDevice sharedDevice] fb_setDeviceInterfaceOrientation:orientation]; diff --git a/docs/multiple-displays.md b/docs/multiple-displays.md index b338b6ae2e..608816ff82 100644 --- a/docs/multiple-displays.md +++ b/docs/multiple-displays.md @@ -22,11 +22,16 @@ specified when creating the session retains its existing priority. W3C touch actions target the selected display. Viewport, element, and pointer origins can be mixed within one action sequence; element offsets remain relative -to the element's center. Element scrolling uses the visible area in the same -coordinate space as the element frame, including on a rotated secondary display. +to the element's center. Element origins require `currentDisplayId` to match the +element's display; they do not override the selected display. Key-only and +pause-only sequences do not require the selected display to be available. +Element scrolling uses the visible area in the same +coordinate space as the element frame, including on rotated main and secondary displays. WDA scrolling rejects an unavailable selected display instead of using an -uncorrected frame. Each drag uses one converted scrolling frame; subsequent -drags refresh the display and orientation. +uncorrected frame. Each normalized scroll keeps its converted frame and vector +in the same coordinate space across its drags. Keep the device orientation and +fold state stable until the command completes; change the display setting between +commands. Scroll-to-visible takes a fresh parent snapshot for each scroll step. This gesture support builds on [#1269](https://github.com/appium/WebDriverAgent/pull/1269). From 451fe295e225df31f3d5fce5f28cb4a4e0769002 Mon Sep 17 00:00:00 2001 From: Kazuaki Matsuo Date: Tue, 6 Oct 2026 10:17:17 -0700 Subject: [PATCH 20/29] refactor: clarify display validation and action synthesis stages --- .../Categories/XCUIElement+FBScrolling.m | 11 ++-- .../Utilities/FBW3CActionsSynthesizer.m | 54 +++++++++++++------ 2 files changed, 44 insertions(+), 21 deletions(-) diff --git a/WebDriverAgentLib/Categories/XCUIElement+FBScrolling.m b/WebDriverAgentLib/Categories/XCUIElement+FBScrolling.m index 1687934fa2..3f4f3bc855 100644 --- a/WebDriverAgentLib/Categories/XCUIElement+FBScrolling.m +++ b/WebDriverAgentLib/Categories/XCUIElement+FBScrolling.m @@ -38,7 +38,7 @@ #if !TARGET_OS_TV -static XCUIScreen *FBScreenForScrolling(void) +static void FBValidateScreenForScrolling(void) { NSError *error = nil; XCUIScreen *screen = [FBScreen currentScreenWithError:&error]; @@ -49,7 +49,6 @@ reason:error.localizedDescription userInfo:nil]; } - return screen; } static CGRect FBScrollFrameInApplication(CGRect frame, XCUIScreen *screen, @@ -350,28 +349,28 @@ @implementation FBXCElementSnapshotWrapper (FBScrolling) - (BOOL)fb_scrollUpByNormalizedDistance:(CGFloat)distance anchorElement:(XCUIElement *)anchorElement { - FBScreenForScrolling(); + FBValidateScreenForScrolling(); return [self fb_scrollByNormalizedVector:CGVectorMake(0.0, distance) anchorElement:anchorElement error:nil]; } - (BOOL)fb_scrollDownByNormalizedDistance:(CGFloat)distance anchorElement:(XCUIElement *)anchorElement { - FBScreenForScrolling(); + FBValidateScreenForScrolling(); return [self fb_scrollByNormalizedVector:CGVectorMake(0.0, -distance) anchorElement:anchorElement error:nil]; } - (BOOL)fb_scrollLeftByNormalizedDistance:(CGFloat)distance anchorElement:(XCUIElement *)anchorElement { - FBScreenForScrolling(); + FBValidateScreenForScrolling(); return [self fb_scrollByNormalizedVector:CGVectorMake(distance, 0.0) anchorElement:anchorElement error:nil]; } - (BOOL)fb_scrollRightByNormalizedDistance:(CGFloat)distance anchorElement:(XCUIElement *)anchorElement { - FBScreenForScrolling(); + FBValidateScreenForScrolling(); return [self fb_scrollByNormalizedVector:CGVectorMake(-distance, 0.0) anchorElement:anchorElement error:nil]; } diff --git a/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m b/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m index 9e89fb4838..3ec387b141 100644 --- a/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m +++ b/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m @@ -870,11 +870,10 @@ @implementation FBW3CActionsSynthesizer return [self eventPathsWithKeyAction:actionDescription forActionId:actionId error:error]; } -- (nullable XCSynthesizedEventRecord *)synthesizeWithError:(NSError **)error +- (nullable NSArray *> *)validatedActionsWithError:(NSError **)error { - BOOL requiresDisplay = NO; NSMutableDictionary *> *actionsMapping = [NSMutableDictionary new]; - NSMutableArray *actionIds = [NSMutableArray new]; + NSMutableArray *> *validatedActions = [NSMutableArray new]; for (NSDictionary *action in self.actions) { id actionId = [action objectForKey:FB_KEY_ID]; if (![actionId isKindOfClass:NSString.class] || 0 == [actionId length]) { @@ -904,21 +903,33 @@ - (nullable XCSynthesizedEventRecord *)synthesizeWithError:(NSError **)error continue; } - if ([action[FB_KEY_TYPE] isEqual:FB_ACTION_TYPE_POINTER]) { - for (NSDictionary *item in actionItems) { - if (![item[FB_ACTION_ITEM_KEY_TYPE] isEqual:FB_ACTION_ITEM_TYPE_PAUSE]) { - requiresDisplay = YES; - break; - } + [validatedActions addObject:action]; + [actionsMapping setObject:action forKey:actionId]; + } + return validatedActions.copy; +} + +- (BOOL)actionsRequireDisplay:(NSArray *> *)actions +{ + for (NSDictionary *action in actions) { + if (![action[FB_KEY_TYPE] isEqual:FB_ACTION_TYPE_POINTER]) { + continue; + } + for (NSDictionary *item in action[FB_KEY_ACTIONS]) { + if (![item[FB_ACTION_ITEM_KEY_TYPE] isEqual:FB_ACTION_ITEM_TYPE_PAUSE]) { + return YES; } } - - [actionIds addObject:actionId]; - [actionsMapping setObject:action forKey:actionId]; } + return NO; +} + +- (nullable XCSynthesizedEventRecord *)eventRecordForActions:(NSArray *> *)actions + error:(NSError **)error +{ // Keyboard and pause-only sequences have no display coordinates. A display // disappearing must not prevent these input sources from being synthesized. - XCUIScreen *screen = requiresDisplay ? [FBScreen currentScreenWithError:error] : XCUIScreen.mainScreen; + XCUIScreen *screen = [self actionsRequireDisplay:actions] ? [FBScreen currentScreenWithError:error] : XCUIScreen.mainScreen; if (nil == screen) { return nil; } @@ -947,8 +958,21 @@ - (nullable XCSynthesizedEventRecord *)synthesizeWithError:(NSError **)error CGSizeMake(screen.bounds.size.width / screen.scale, screen.bounds.size.height / screen.scale), eventRecord.interfaceOrientation); } - for (NSString *actionId in actionIds.copy) { - NSDictionary *actionDescription = [actionsMapping objectForKey:actionId]; + return eventRecord; +} + +- (nullable XCSynthesizedEventRecord *)synthesizeWithError:(NSError **)error +{ + NSArray *> *actions = [self validatedActionsWithError:error]; + if (nil == actions) { + return nil; + } + XCSynthesizedEventRecord *eventRecord = [self eventRecordForActions:actions error:error]; + if (nil == eventRecord) { + return nil; + } + for (NSDictionary *actionDescription in actions) { + NSString *actionId = actionDescription[FB_KEY_ID]; NSArray *eventPaths = [self eventPathsWithActionDescription:actionDescription forActionId:actionId error:error]; if (nil == eventPaths) { return nil; From 947cc2a9cccacab97bf849040efd792d2b9534ca Mon Sep 17 00:00:00 2001 From: Kazuaki Matsuo Date: Tue, 6 Oct 2026 10:19:49 -0700 Subject: [PATCH 21/29] test: preserve shared alert fixture in display-independent action test --- .../IntegrationTests/FBW3CTouchActionsIntegrationTests.m | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/WebDriverAgentTests/IntegrationTests/FBW3CTouchActionsIntegrationTests.m b/WebDriverAgentTests/IntegrationTests/FBW3CTouchActionsIntegrationTests.m index 020e5773ae..f0ae86ecb8 100644 --- a/WebDriverAgentTests/IntegrationTests/FBW3CTouchActionsIntegrationTests.m +++ b/WebDriverAgentTests/IntegrationTests/FBW3CTouchActionsIntegrationTests.m @@ -38,7 +38,8 @@ @implementation FBW3CTouchActionsIntegrationTestsPart1 - (void)testDisplayIndependentActionsAfterSelectedDisplayDisappears { - [self launchApplication]; + // setUp already opens the Alerts fixture; relaunching would reset that page + // and break the subsequent touch tests that share it. NSNumber *previous = FBConfiguration.sharedInstance.currentDisplayId; [self addTeardownBlock:^{ FBConfiguration.sharedInstance.currentDisplayId = previous; }]; NSArray *sequences = @[ From 5ed90c81840e42326211ddfabd881934addd5dca Mon Sep 17 00:00:00 2001 From: Kazuaki Matsuo Date: Tue, 6 Oct 2026 20:43:11 -0700 Subject: [PATCH 22/29] fix: anchor native Duo coordinates to application windows (#1292) * fix: anchor native Duo coordinates to application windows * test: cover native coordinates across displays and orientations * fix: restrict native coordinates to windows on the selected display * refactor: use guards for native gesture coordinate routing --- .../Commands/FBElementCommands.m | 29 ++++- .../IntegrationApp/COORDINATE_PROBE.md | 1 + .../Classes/FBCoordinateProbeViewController.m | 11 ++ .../IntegrationTests/FBDisplayGestureTests.m | 117 ++++++++++++++++++ docs/multiple-displays.md | 3 + 5 files changed, 160 insertions(+), 1 deletion(-) diff --git a/WebDriverAgentLib/Commands/FBElementCommands.m b/WebDriverAgentLib/Commands/FBElementCommands.m index 97afb7b6e0..744f1322ac 100644 --- a/WebDriverAgentLib/Commands/FBElementCommands.m +++ b/WebDriverAgentLib/Commands/FBElementCommands.m @@ -9,6 +9,8 @@ #import "FBElementCommands.h" #import "FBConfiguration.h" +#import "FBScreen.h" +#import "XCUIScreen.h" #import "FBKeyboard.h" #import "FBRoute.h" #import "FBRouteRequest.h" @@ -698,7 +700,32 @@ + (nullable XCUICoordinate *)gestureCoordinateWithOffset:(CGVector)offset element:(XCUIElement *)element error:(NSError **)error { - return FBCoordinateWithAnchorOffset(element, CGVectorMake(0, 0), offset, error); + if (element.elementType != XCUIElementTypeApplication) { + return FBCoordinateWithAnchorOffset(element, CGVectorMake(0, 0), offset, error); + } + XCUIScreen *screen = [FBScreen currentScreenWithError:error]; + if (nil == screen) { + return nil; + } + if (screen.displayID == XCUIScreen.mainScreen.displayID) { + return FBCoordinateWithAnchorOffset(element, CGVectorMake(0, 0), offset, error); + } + // Application-root coordinates are rotated using the main display on Duo. + // A window coordinate keeps XCTest's display affinity and rotation intact. + CGPoint point = CGPointMake(offset.dx, offset.dy); + for (XCUIElement *window in [element childrenMatchingType:XCUIElementTypeWindow].allElementsBoundByIndex) { + // Resolve the window before reading its display affinity: an unresolved + // element may report displayID == 0. + (void)window.frame; + CGRect frame = window.wdFrame; + if (window.screen.displayID == screen.displayID && CGRectContainsPoint(frame, point)) { + return FBCoordinateWithAnchorOffset(window, CGVectorMake(0, 0), + CGVectorMake(point.x - frame.origin.x, point.y - frame.origin.y), error); + } + } + [[[FBErrorBuilder builder] withDescriptionFormat:@"No application window on display %lld contains the requested coordinate", screen.displayID] + buildError:error]; + return nil; } /** diff --git a/WebDriverAgentTests/IntegrationApp/COORDINATE_PROBE.md b/WebDriverAgentTests/IntegrationApp/COORDINATE_PROBE.md index 87a21eff23..7f3fd73f41 100644 --- a/WebDriverAgentTests/IntegrationApp/COORDINATE_PROBE.md +++ b/WebDriverAgentTests/IntegrationApp/COORDINATE_PROBE.md @@ -23,6 +23,7 @@ The `probe-status` label exposes `start`, `last`, `phase`, and `count` after a touch. `start` and `last` are **canvas-local app points**. It also exposes `canvasBounds`, `canvasWindowRect`, `windowSize`, `screenSize`, `scrollY`, and `fullyVisibleRows` (row indices whose full rectangles lie inside the table bounds). +`rotationInProgress` stays true until the fixture's rotation transition completes. Geometry is refreshed when the view lays out, including after rotation or window resizing. Touch-related fields are absent before the first touch. diff --git a/WebDriverAgentTests/IntegrationApp/Classes/FBCoordinateProbeViewController.m b/WebDriverAgentTests/IntegrationApp/Classes/FBCoordinateProbeViewController.m index f407129ca6..96d340458d 100644 --- a/WebDriverAgentTests/IntegrationApp/Classes/FBCoordinateProbeViewController.m +++ b/WebDriverAgentTests/IntegrationApp/Classes/FBCoordinateProbeViewController.m @@ -106,6 +106,17 @@ - (void)viewDidLoad [self.view addSubview:self.table]; } +- (void)viewWillTransitionToSize:(CGSize)size withTransitionCoordinator:(id)coordinator +{ + [super viewWillTransitionToSize:size withTransitionCoordinator:coordinator]; + self.measurement[@"rotationInProgress"] = @YES; + [self publishMeasurement]; + [coordinator animateAlongsideTransition:nil completion:^(id context) { + self.measurement[@"rotationInProgress"] = @NO; + [self publishMeasurement]; + }]; +} + - (void)viewDidLayoutSubviews { [super viewDidLayoutSubviews]; diff --git a/WebDriverAgentTests/IntegrationTests/FBDisplayGestureTests.m b/WebDriverAgentTests/IntegrationTests/FBDisplayGestureTests.m index 2328fdbe8a..39589afab3 100644 --- a/WebDriverAgentTests/IntegrationTests/FBDisplayGestureTests.m +++ b/WebDriverAgentTests/IntegrationTests/FBDisplayGestureTests.m @@ -11,7 +11,13 @@ #import "FBIntegrationTestCase.h" #import "FBConfiguration.h" #import "FBScreen.h" +#import "FBRunLoopSpinner.h" #import "FBExceptions.h" +#import "FBElementCommands.h" + +@interface FBElementCommands (CoordinateTests) ++ (nullable XCUICoordinate *)gestureCoordinateWithOffset:(CGVector)offset element:(XCUIElement *)element error:(NSError **)error; +@end #import "XCUIApplication+FBTouchAction.h" #import "XCUIElement+FBScrolling.h" @@ -104,6 +110,117 @@ - (NSDictionary *)moveFrom:(id)origin x:(CGFloat)x y:(CGFloat)y duration:(NSUInt return @{@"type": @"pointerMove", @"origin": origin, @"x": @(x), @"y": @(y), @"duration": @(duration)}; } +- (void)verifyNativeApplicationCoordinates +{ + NSDictionary *geometry = [self measurement]; + NSArray *bounds = geometry[@"canvasBounds"]; + NSArray *frame = geometry[@"canvasWindowRect"]; + for (NSNumber *x in @[@40, @(bounds[0].doubleValue - 40)]) { + for (NSNumber *y in @[@40, @(bounds[1].doubleValue - 40)]) { + NSUInteger count = [[self measurement][@"count"] unsignedIntegerValue]; + NSError *error = nil; + XCUICoordinate *coordinate = [FBElementCommands gestureCoordinateWithOffset: + CGVectorMake(frame[0].doubleValue + x.doubleValue, frame[1].doubleValue + y.doubleValue) + element:self.testedApplication error:&error]; + XCTAssertNotNil(coordinate); + XCTAssertNil(error); + [coordinate tap]; + NSDictionary *result = [self measurement]; + XCTAssertEqual([result[@"count"] unsignedIntegerValue], count + 1); + XCTAssertEqualWithAccuracy([result[@"last"][0] doubleValue], x.doubleValue, 1); + XCTAssertEqualWithAccuracy([result[@"last"][1] doubleValue], y.doubleValue, 1); + } + } +} + +- (void)testNativeApplicationCoordinates +{ + [self verifyNativeApplicationCoordinates]; +} + +- (void)testNativeMainDisplayDefaultCoordinates +{ + XCTSkipIf(FBConfiguration.sharedInstance.currentDisplayId.longLongValue != [FBScreen displayID], + @"Requires the fixture on the main display"); + FBConfiguration.sharedInstance.currentDisplayId = nil; + [self verifyNativeApplicationCoordinates]; + FBConfiguration.sharedInstance.currentDisplayId = @([FBScreen displayID]); + [self verifyNativeApplicationCoordinates]; +} + +- (void)verifyNativeCoordinatesInOrientation:(UIDeviceOrientation)orientation +{ + XCTSkipIf(XCUIDevice.sharedDevice.fb_canAttemptSimulatedHingeAngleInjection, + @"Foldable orientations are covered by testNativeCoordinatesAcrossFoldStates"); + UIDeviceOrientation previousOrientation = XCUIDevice.sharedDevice.orientation; + [self addTeardownBlock:^{ + [XCUIDevice.sharedDevice fb_setDeviceInterfaceOrientation:previousOrientation]; + }]; + XCTAssertTrue([XCUIDevice.sharedDevice fb_setDeviceInterfaceOrientation:orientation]); + XCTAssertTrue([[[FBRunLoopSpinner new] timeout:5] spinUntilTrue:^BOOL { + NSDictionary *geometry = [self measurement]; + NSArray *size = geometry[@"windowSize"]; + return size[0].doubleValue > size[1].doubleValue + && ![geometry[@"rotationInProgress"] boolValue]; + }]); + [self verifyNativeApplicationCoordinates]; +} + +- (void)testNativeCoordinatesInLandscapeLeft +{ + [self verifyNativeCoordinatesInOrientation:UIDeviceOrientationLandscapeLeft]; +} + +- (void)testNativeCoordinatesInLandscapeRight +{ + [self verifyNativeCoordinatesInOrientation:UIDeviceOrientationLandscapeRight]; +} + +- (void)testNativeCoordinatesRejectMismatchedWindowDisplay +{ + NSNumber *fixtureDisplay = FBConfiguration.sharedInstance.currentDisplayId; + NSArray *frame = [self measurement][@"canvasWindowRect"]; + BOOL checkedSecondaryDisplay = NO; + for (NSDictionary *screen in [FBScreen screensWithError:nil]) { + if ([screen[@"isMain"] boolValue] || [screen[@"displayId"] isEqual:fixtureDisplay]) { + continue; + } + checkedSecondaryDisplay = YES; + FBConfiguration.sharedInstance.currentDisplayId = screen[@"displayId"]; + NSError *error = nil; + XCUICoordinate *coordinate = [FBElementCommands gestureCoordinateWithOffset: + CGVectorMake(frame[0].doubleValue + frame[2].doubleValue / 2, + frame[1].doubleValue + frame[3].doubleValue / 2) + element:self.testedApplication error:&error]; + XCTAssertNil(coordinate, @"A window on display %@ must not satisfy a request for display %@", + fixtureDisplay, screen[@"displayId"]); + XCTAssertNotNil(error); + } + XCTSkipIf(!checkedSecondaryDisplay, @"Requires an available secondary display without the fixture"); +} + +- (void)testNativeCoordinatesAcrossFoldStates +{ + XCUIDevice *device = XCUIDevice.sharedDevice; + XCTSkipIf(!device.fb_canAttemptSimulatedHingeAngleInjection, @"Requires a foldable simulator"); + NSError *error = nil; + NSNumber *originalAngle = [device fb_getSimulatedHingeAngle:&error]; + XCTAssertNotNil(originalAngle, @"%@", error); + [self addTeardownBlock:^{ + if (nil != originalAngle) { + [device fb_setSimulatedHingeAngle:originalAngle.doubleValue error:nil]; + } + }]; + for (NSNumber *angle in @[@0, @90, @180, @0]) { + [self.testedApplication terminate]; + XCTAssertTrue([device fb_setSimulatedHingeAngle:angle.doubleValue error:&error], @"%@", error); + [self launchApplication]; + [self.testedApplication.buttons[@"coordinate-probe"] tap]; + [self selectFixtureDisplay]; + [self verifyNativeApplicationCoordinates]; + } +} + - (void)testViewportAndElementCorners { NSDictionary *geometry = [self measurement]; diff --git a/docs/multiple-displays.md b/docs/multiple-displays.md index 608816ff82..afbe26b690 100644 --- a/docs/multiple-displays.md +++ b/docs/multiple-displays.md @@ -33,6 +33,9 @@ in the same coordinate space across its drags. Keep the device orientation and fold state stable until the command completes; change the display setting between commands. Scroll-to-visible takes a fresh parent snapshot for each scroll step. +Native application-root coordinates on a secondary display require a containing +application window on that display. Requests outside those windows fail explicitly. + This gesture support builds on [#1269](https://github.com/appium/WebDriverAgent/pull/1269). ## Validation and feedback From 1b82530b7529e35c6c0c5023678e799e5c5afa11 Mon Sep 17 00:00:00 2001 From: Kazuaki Matsuo Date: Tue, 6 Oct 2026 22:14:23 -0700 Subject: [PATCH 23/29] docs: document legacy SDK coordinate limitations on Duo --- .../Utilities/FBW3CActionsSynthesizer.m | 7 ++- docs/multiple-displays.md | 48 +++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m b/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m index 3ec387b141..ed048b02b1 100644 --- a/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m +++ b/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m @@ -294,7 +294,12 @@ - (nullable XCUICoordinate *)positionWithError:(NSError **)error } if (nil != element) { - // Normalized element coordinates already resolve against their own display. + // Normalized element coordinates resolve against the element's own display. + // This assumes XCTest can resolve that app's coordinates correctly. On the + // Duo iOS 27.1 simulator, an SDK 26.5 compatibility app misses inner-display + // taps even in pure XCTest without WDA; the same app built with SDK 27.2 works. + // Do not apply the viewport offset as a workaround for this separate issue. + // See docs/multiple-displays.md#legacy-sdk-compatibility-windows for evidence. self.usesDisplayCorrection = NO; if (nil == x && nil == y) { return [self hitpointWithElement:element positionOffset:nil error:error]; diff --git a/docs/multiple-displays.md b/docs/multiple-displays.md index 06f1ae1aef..afd9d877fb 100644 --- a/docs/multiple-displays.md +++ b/docs/multiple-displays.md @@ -46,6 +46,54 @@ application window on that display. Requests outside those windows fail explicit This gesture support builds on [#1269](https://github.com/appium/WebDriverAgent/pull/1269). +## Legacy SDK compatibility windows + +Older-SDK apps can run in a compatibility window on iPhone Duo. Selecting the +correct `currentDisplayId` is still required, but it does not address failures +in XCTest's handling of that compatibility layout. This is separate from sending +a gesture to the wrong display. + +A controlled comparison on the **iPhone Duo simulator, iOS 27.1 (24A94401)** +reproduced missed input without WDA. The same coordinate-probe app source was built +with **SDK 26.5 (Xcode 26.6)** and **SDK 27.2 (Xcode 27.2 beta 2)**, using separate +bundle IDs. Both were driven by the same Xcode 27.2 beta 2 UI-test runner. The +runner did not link WDA, and checked that WDA's `FBScreen` class was absent. +Only pose setup used simulated hinge injection; measured taps used public +`XCUIElement.tap()` and `coordinate(withNormalizedOffset:).tap()` APIs. + +Each configuration tested the element center and four normalized positions. +The app recorded actual canvas-local touch coordinates and a touch counter; +returning from a tap without an XCTest error did not count as successful input. +Closed and fully open poses were repeated with the app order reversed, and the +second run also covered a 90-degree pose: + +| Duo pose | SDK 26.5: touches received / attempted | SDK 27.2: touches received / attempted | +| --- | --- | --- | +| Closed (0 degrees) | 10 / 10 | 10 / 10 | +| Fully open (180 degrees) | 0 / 10 | 10 / 10 | +| Book (90 degrees) | 0 / 5 | 5 / 5 | + +Successful taps differed from the expected canvas-local position by less than +0.17 points on either axis. The old-SDK app reported a 375 x 667 logical window +in both closed and open poses. On the inner display, XCTest reported the canvas +frame as approximately `(469, 6.67, 136.33, 111.67)`, while the app's own window +coordinates were `(20, 218, 335, 409)`. Different coordinate spaces can legitimately +have different frames; the evidence of failure is the missing delivered touches, +not the frame difference alone. + +This isolates the observed failure to the Apple compatibility-display, +accessibility, or XCTest path rather than WDA's gesture-coordinate calculations. +It does not identify the failing internal Apple component. Results are limited +to this simulator/runtime, fixture, and SDK pair; physical Duo behavior and other +SDK combinations remain unverified. Rebuilding the fixture with SDK 27.2 avoided +the failure in this comparison, but is not a guarantee for every app. + +WDA does not apply a speculative coordinate correction for compatibility windows. +When diagnosing a similar failure, first select the app's display explicitly, +then compare direct XCTest taps and app-recorded input before changing coordinate +transforms. For Apple's SDK-dependent layout guidance, see +[Prepare your app for iPhone Duo](https://developer.apple.com/videos/play/tech-talks/111461/). + ## Validation and feedback Duo display behavior has been tested on the iPhone Duo simulator with Xcode 27.1. From 56a9dd5a4ac49b47d11983cd80ba2955485f728e Mon Sep 17 00:00:00 2001 From: Kazuaki Matsuo Date: Tue, 6 Oct 2026 23:23:20 -0700 Subject: [PATCH 24/29] test: wait for rotated gesture fixture geometry to settle --- .../IntegrationTests/FBW3CTouchActionsIntegrationTests.m | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/WebDriverAgentTests/IntegrationTests/FBW3CTouchActionsIntegrationTests.m b/WebDriverAgentTests/IntegrationTests/FBW3CTouchActionsIntegrationTests.m index f0ae86ecb8..43c124dba9 100644 --- a/WebDriverAgentTests/IntegrationTests/FBW3CTouchActionsIntegrationTests.m +++ b/WebDriverAgentTests/IntegrationTests/FBW3CTouchActionsIntegrationTests.m @@ -60,6 +60,12 @@ - (void)testDisplayIndependentActionsAfterSelectedDisplayDisappears - (void)verifyGesture:(NSArray *> *)gesture orientation:(UIDeviceOrientation)orientation { [[XCUIDevice sharedDevice] fb_setDeviceInterfaceOrientation:orientation]; + // XCTest may report the new interface orientation while the Alerts button + // still has transposed geometry from the rotation transition. The fixture's + // button is wider than it is tall in every settled orientation. Wait for that + // layout before resolving the W3C action's element-origin coordinates. + XCUIElement *button = self.testedApplication.buttons[FBShowAlertButtonName]; + FBAssertWaitTillBecomesTrue(CGRectGetWidth(button.frame) > CGRectGetHeight(button.frame)); NSError *error; XCTAssertTrue([self.testedApplication fb_performW3CActions:gesture elementCache:nil error:&error]); FBAssertWaitTillBecomesTrue(self.testedApplication.alerts.count > 0); From 44cfddc3d3e1a4216ddb4bc6a5f93dac5334975f Mon Sep 17 00:00:00 2001 From: Kazuaki Matsuo Date: Tue, 6 Oct 2026 23:26:10 -0700 Subject: [PATCH 25/29] docs: record verified Duo simulator capture and layout limitations --- docs/multiple-displays.md | 44 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/docs/multiple-displays.md b/docs/multiple-displays.md index afd9d877fb..c0d0e5bb4d 100644 --- a/docs/multiple-displays.md +++ b/docs/multiple-displays.md @@ -5,8 +5,9 @@ Set `currentDisplayId` through `POST /session/:sessionId/appium/settings` to sel one of these IDs. Do not assume that the main display is the currently visible one after folding or unfolding the device. -The setting selects the display for screenshots, MJPEG frames, and newly started -XCTest screen recordings. MJPEG follows setting changes on the existing +The setting selects the display for screenshots and MJPEG frames, and passes the +selected display ID to newly started XCTest screen recordings (see the +[simulator recording limitation](#xctest-recording-on-the-duo-simulator) below). MJPEG follows setting changes on the existing connection; an in-progress recording keeps the display selected when it started. An unavailable display causes screenshots and recording requests to fail; MJPEG pauses frame delivery until a valid display is selected. Set `currentDisplayId` @@ -94,9 +95,46 @@ then compare direct XCTest taps and app-recorded input before changing coordinat transforms. For Apple's SDK-dependent layout guidance, see [Prepare your app for iPhone Duo](https://developer.apple.com/videos/play/tech-talks/111461/). +## XCTest recording on the Duo simulator + +On **iOS 27.1 (24A94401), with Xcode 27.2 beta 2**, the native XCTest +recording path produced black video for the closed Duo's outer display. +This reproduced after starting WDA afresh while the device was closed, as well +as after switching from the inner display back to the outer display. + +The recorded native `XCTScreenRecordingRequest.screenID` was `1` for the outer +screen and `3` for the inner screen, matching the selected IDs returned by +`GET /wda/screens`. Simultaneous outer-display screenshots were correct at +1398 x 2034 pixels. Nevertheless, each four-second outer recording in the fresh +runner experiment contained only one black frame at 2006 x 2852 pixels; the +inner-display recording contained visible content. A successful start/stop +response is therefore insufficient to verify captured video. + +The failure occurs downstream of the selected ID being supplied to XCTest; +this experiment does not identify the failing internal Apple component or +establish behavior on other runtimes or physical devices. WDA does not remap +display IDs to work around it. Use the MJPEG/ffmpeg recording path when affected; +that path was verified across closed, book, and fully open poses with both MJPEG +and H.264 output. Set `mjpegFixOrientation` to `true` if upright encoded MJPEG +pixels are required: the default preserves the existing EXIF-oriented output. + +## Waiting for rotated layouts + +An application's reported interface orientation can update before its element +frames settle. On an iPad simulator running iOS 26.5, the gesture fixture's Alerts +button briefly retained transposed dimensions immediately after rotation; +resolving an element-origin gesture during that interval missed the button. +The same failure reproduced on the unmodified WDA baseline. Waiting for the +fixture's expected landscape geometry resolved it; direct XCTest gesture +controls also passed. Tests should wait for the application's expected layout +before resolving coordinates, in addition to keeping its pose stable during a +command. This observation does not imply that every missed gesture is an +XCTest defect. + ## Validation and feedback -Duo display behavior has been tested on the iPhone Duo simulator with Xcode 27.1. +Duo display behavior has been tested on the iPhone Duo simulator with Xcode 27.1 +and Xcode 27.2 beta 2. We have not had access to a physical Duo for testing. If you have access to one, we would appreciate [reports of its display and capture behavior](https://github.com/appium/WebDriverAgent/issues), including the device model, iOS/Xcode and WDA versions, selected display IDs, From c0765edd59ac26584a6dfb201ffaef5e12ad2662 Mon Sep 17 00:00:00 2001 From: Kazuaki Matsuo Date: Wed, 7 Oct 2026 00:01:03 -0700 Subject: [PATCH 26/29] fix: normalize portrait geometry for landscape-native screens --- WebDriverAgentLib/Commands/FBCustomCommands.m | 9 ++++++++- .../IntegrationTests/FBScreenTests.m | 20 ++++++++++++------- docs/multiple-displays.md | 16 ++++++++++++++- 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/WebDriverAgentLib/Commands/FBCustomCommands.m b/WebDriverAgentLib/Commands/FBCustomCommands.m index 4273242be2..72d5593334 100644 --- a/WebDriverAgentLib/Commands/FBCustomCommands.m +++ b/WebDriverAgentLib/Commands/FBCustomCommands.m @@ -174,7 +174,14 @@ + (NSArray *)routes CGRectGetHeight(screen.bounds) / screen.scale); #if !TARGET_OS_TV && !TARGET_OS_WATCH XCUIApplication *activeApp = request.session ? request.session.activeApplication : XCUIApplication.fb_activeApplication; - screenSize = FBAdjustDimensionsForApplication(screenSize, activeApp.interfaceOrientation); + UIInterfaceOrientation orientation = activeApp.interfaceOrientation; + // Screen bounds describe the panel's native orientation. Some physical iPads + // expose landscape-native bounds even while the application is in portrait. + // Normalize the full screen separately from compatibility-mode app frames. + if (UIInterfaceOrientationIsPortrait(orientation) && screenSize.width > screenSize.height) { + screenSize = CGSizeMake(screenSize.height, screenSize.width); + } + screenSize = FBAdjustDimensionsForApplication(screenSize, orientation); #endif // SpringBoard can expose hidden status bars belonging to other displays, and diff --git a/WebDriverAgentTests/IntegrationTests/FBScreenTests.m b/WebDriverAgentTests/IntegrationTests/FBScreenTests.m index e63d0351b1..e479a22726 100644 --- a/WebDriverAgentTests/IntegrationTests/FBScreenTests.m +++ b/WebDriverAgentTests/IntegrationTests/FBScreenTests.m @@ -362,7 +362,7 @@ - (void)testScreenInfoRejectsUnavailableDisplay XCTAssertTrue([info[@"message"] containsString:@"Available display ids"]); } -- (void)testScreenInfoInBothLandscapeOrientations +- (void)testScreenInfoAcrossPortraitAndLandscapeOrientations { XCTSkipIf(XCUIDevice.sharedDevice.fb_canAttemptSimulatedHingeAngleInjection, @"Foldable orientations are covered by testScreenInfoAcrossFoldStates"); @@ -374,25 +374,31 @@ - (void)testScreenInfoInBothLandscapeOrientations [[XCUIDevice sharedDevice] fb_setDeviceInterfaceOrientation:UIDeviceOrientationPortrait]; }]; BOOL expectsVisibleBar = UIDevice.currentDevice.userInterfaceIdiom == UIUserInterfaceIdiomPad; - for (NSNumber *orientation in @[@(UIDeviceOrientationLandscapeLeft), @(UIDeviceOrientationLandscapeRight)]) { + for (NSNumber *orientation in @[@(UIDeviceOrientationPortrait), + @(UIDeviceOrientationLandscapeLeft), + @(UIDeviceOrientationLandscapeRight), + @(UIDeviceOrientationPortrait)]) { XCTAssertTrue([[XCUIDevice sharedDevice] fb_setDeviceInterfaceOrientation:orientation.integerValue]); __block NSDictionary *info = nil; - // iPad keeps its top status bar in landscape. Wait for rotation to finish - // rather than accepting a transient zero crop as a successful result. + BOOL isPortrait = UIDeviceOrientationIsPortrait(orientation.integerValue); + // Include portrait before and after rotation: landscape-native iPad panels + // must swap their screen bounds back and retain the visible top status bar. + // Wait for rotation rather than accepting a transient zero crop. XCTAssertTrue([[[FBRunLoopSpinner new] timeout:5] spinUntilTrue:^BOOL { info = [self screenInfoResponse]; double width = [info[@"screenSize"][@"width"] doubleValue]; double height = [info[@"screenSize"][@"height"] doubleValue]; double barWidth = [info[@"statusBarSize"][@"width"] doubleValue]; double barHeight = [info[@"statusBarSize"][@"height"] doubleValue]; - if (nil != info[@"error"] || width <= height || height <= 0) { + if (nil != info[@"error"] || width <= 0 || height <= 0 + || (isPortrait ? width >= height : width <= height)) { return NO; } if (barHeight == 0) { - return !expectsVisibleBar && barWidth == 0; + return !expectsVisibleBar && !isPortrait && barWidth == 0; } return barHeight > 0 && barHeight < height && fabs(barWidth - width) <= 1; - }], @"Unexpected landscape screen metadata: %@", info); + }], @"Unexpected screen metadata at orientation %@: %@", orientation, info); } } diff --git a/docs/multiple-displays.md b/docs/multiple-displays.md index c0d0e5bb4d..6cc168b061 100644 --- a/docs/multiple-displays.md +++ b/docs/multiple-displays.md @@ -72,7 +72,7 @@ second run also covered a 90-degree pose: | --- | --- | --- | | Closed (0 degrees) | 10 / 10 | 10 / 10 | | Fully open (180 degrees) | 0 / 10 | 10 / 10 | -| Book (90 degrees) | 0 / 5 | 5 / 5 | +| Book (90 degrees, approached from fully open) | 0 / 5 | 5 / 5 | Successful taps differed from the expected canvas-local position by less than 0.17 points on either axis. The old-SDK app reported a 375 x 667 logical window @@ -89,6 +89,20 @@ to this simulator/runtime, fixture, and SDK pair; physical Duo behavior and othe SDK combinations remain unverified. Rebuilding the fixture with SDK 27.2 avoided the failure in this comparison, but is not a guarantee for every app. +A further round trip exercised `0 → 45 → 90 → 135 → 180 → 90 → 0` +degrees, both with a fresh app launch at each pose and with the same app kept +running. Each pose used three direct XCTest taps and attempted to switch to a +scrollable table. The new-SDK fixture received all 42 taps. The old-SDK fixture +received 24 of 42: taps worked at 0, 45, and 90 degrees while opening and after +returning to 0, but none arrived at 135, 180, or 90 degrees while closing. In +those failing states, the direct XCTest tap could not even select the Scroll +segment. The old-SDK fixture entered scroll mode and scrolled in all eight +remaining cases. The new-SDK fixture entered scroll mode in all 14 cases; +12 swipes changed the offset and the last two warm-run swipes were already at +the end of the table. These observations cover both cold and warm transitions; +they do not imply that a hinge angle alone determines the active display or +whether input succeeds. In particular, the two 90-degree states differed. + WDA does not apply a speculative coordinate correction for compatibility windows. When diagnosing a similar failure, first select the app's display explicitly, then compare direct XCTest taps and app-recorded input before changing coordinate From 5f4fa775e83fd40e0c70f7487570f34fb7061f0a Mon Sep 17 00:00:00 2001 From: Kazuaki Matsuo Date: Wed, 7 Oct 2026 07:55:59 -0700 Subject: [PATCH 27/29] fix: align application origins on secondary displays --- .../Utilities/FBW3CActionsSynthesizer.m | 26 +++++- .../IntegrationTests/FBDisplayGestureTests.m | 85 ++++++++++++++++++- docs/multiple-displays.md | 8 ++ 3 files changed, 116 insertions(+), 3 deletions(-) diff --git a/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m b/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m index ed048b02b1..79dc3c6a73 100644 --- a/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m +++ b/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m @@ -172,6 +172,27 @@ - (nullable XCUICoordinate *)hitpointWithElement:(nullable XCUIElement *)element positionOffset:(nullable NSValue *)positionOffset error:(NSError **)error { + XCUIScreen *screen = nil; + if (element.elementType == XCUIElementTypeApplication) { + screen = [FBScreen currentScreenWithError:error]; + if (nil == screen) { + return nil; + } + } + if (nil != screen && screen.displayID != XCUIScreen.mainScreen.displayID) { + // Unlike ordinary elements, application-root coordinates rotate using the + // main display. Resolve the client-visible rect's center into viewport + // coordinates, then let screenPoint apply the selected-display correction. + // Do not use a normalized app center: XCTest may report a portrait app rect + // for a landscape window, so its normalized center uses a different space. + CGRect frame = element.wdFrame; + if (CGRectIsEmpty(frame)) { + return [super hitpointWithElement:element positionOffset:positionOffset error:error]; + } + CGPoint offset = nil == positionOffset ? CGPointZero : positionOffset.CGPointValue; + CGPoint point = CGPointMake(CGRectGetMidX(frame) + offset.x, CGRectGetMidY(frame) + offset.y); + return [super hitpointWithElement:nil positionOffset:[NSValue valueWithCGPoint:point] error:error]; + } if (nil == element || nil == positionOffset) { return [super hitpointWithElement:element positionOffset:positionOffset error:error]; } @@ -294,13 +315,14 @@ - (nullable XCUICoordinate *)positionWithError:(NSError **)error } if (nil != element) { - // Normalized element coordinates resolve against the element's own display. + // Application origins are converted to viewport coordinates above and need + // its display correction. Ordinary element anchors use their own display. // This assumes XCTest can resolve that app's coordinates correctly. On the // Duo iOS 27.1 simulator, an SDK 26.5 compatibility app misses inner-display // taps even in pure XCTest without WDA; the same app built with SDK 27.2 works. // Do not apply the viewport offset as a workaround for this separate issue. // See docs/multiple-displays.md#legacy-sdk-compatibility-windows for evidence. - self.usesDisplayCorrection = NO; + self.usesDisplayCorrection = element.elementType == XCUIElementTypeApplication; if (nil == x && nil == y) { return [self hitpointWithElement:element positionOffset:nil error:error]; } diff --git a/WebDriverAgentTests/IntegrationTests/FBDisplayGestureTests.m b/WebDriverAgentTests/IntegrationTests/FBDisplayGestureTests.m index 39589afab3..4967b778f1 100644 --- a/WebDriverAgentTests/IntegrationTests/FBDisplayGestureTests.m +++ b/WebDriverAgentTests/IntegrationTests/FBDisplayGestureTests.m @@ -14,6 +14,8 @@ #import "FBRunLoopSpinner.h" #import "FBExceptions.h" #import "FBElementCommands.h" +#import "FBElementCache.h" +#import "XCUIElement+FBWebDriverAttributes.h" @interface FBElementCommands (CoordinateTests) + (nullable XCUICoordinate *)gestureCoordinateWithOffset:(CGVector)offset element:(XCUIElement *)element error:(NSError **)error; @@ -27,6 +29,7 @@ + (nullable XCUICoordinate *)gestureCoordinateWithOffset:(CGVector)offset elemen @interface FBDisplayGestureTests : FBIntegrationTestCase @property (nonatomic) NSNumber *previousDisplayId; @property (nonatomic) XCUIElement *canvas; +@property (nonatomic) FBElementCache *actionElementCache; @end @implementation FBDisplayGestureTests @@ -35,6 +38,7 @@ - (void)setUp { [super setUp]; self.previousDisplayId = FBConfiguration.sharedInstance.currentDisplayId; + self.actionElementCache = nil; if (!XCUIDevice.sharedDevice.fb_canAttemptSimulatedHingeAngleInjection) { [self resetOrientation]; } @@ -96,8 +100,9 @@ - (void)performMoves:(NSArray *)moves expectedEnd:(CGPoint)expec NSArray *actions = @[@{ @"type": @"pointer", @"id": @"finger", @"parameters": @{@"pointerType": @"touch"}, @"actions": items, }]; - XCTAssertTrue([self.testedApplication fb_performW3CActions:actions elementCache:nil error:&error], @"%@", error); + XCTAssertTrue([self.testedApplication fb_performW3CActions:actions elementCache:self.actionElementCache error:&error], @"%@", error); NSDictionary *result = [self measurement]; + NSLog(@"W3C_DELIVERED expected=%@ actual=%@ count=%@ phase=%@", NSStringFromCGPoint(expected), result[@"last"], result[@"count"], result[@"phase"]); XCTAssertEqual([result[@"count"] unsignedIntegerValue], count + 1); XCTAssertEqualObjects(result[@"phase"], @"ended"); NSArray *actual = result[@"last"]; @@ -107,6 +112,11 @@ - (void)performMoves:(NSArray *)moves expectedEnd:(CGPoint)expec - (NSDictionary *)moveFrom:(id)origin x:(CGFloat)x y:(CGFloat)y duration:(NSUInteger)duration { + if (nil != self.actionElementCache && [origin isKindOfClass:XCUIElement.class]) { + NSString *uuid = [self.actionElementCache storeElement:origin]; + XCTAssertNotNil(uuid); + origin = @{@"element-6066-11e4-a52e-4f735466cecf": uuid}; + } return @{@"type": @"pointerMove", @"origin": origin, @"x": @(x), @"y": @(y), @"duration": @(duration)}; } @@ -221,6 +231,79 @@ - (void)testNativeCoordinatesAcrossFoldStates } } +- (void)verifyW3CApplicationOrigin +{ + // Resolve serialized element IDs through the same cache as the HTTP API. + // Passing raw XCTest objects can hide application-root coordinate errors. + self.actionElementCache = [FBElementCache new]; + NSDictionary *geometry = [self measurement]; + NSArray *bounds = geometry[@"canvasBounds"]; + NSArray *frame = geometry[@"canvasWindowRect"]; + CGPoint local = CGPointMake(bounds[0].doubleValue / 2, bounds[1].doubleValue / 2 - 20); + CGPoint target = CGPointMake(frame[0].doubleValue + local.x, frame[1].doubleValue + local.y); + // Use the same rect a client receives, not the window's differently oriented + // frame. On Duo the app rect may still have transposed dimensions. + CGRect appRect = self.testedApplication.wdFrame; + NSDictionary *appMove = [self moveFrom:self.testedApplication + x:target.x - CGRectGetMidX(appRect) + y:target.y - CGRectGetMidY(appRect) duration:0]; + [self performMoves:@[[self moveFrom:@"viewport" x:target.x y:target.y duration:0]] expectedEnd:local]; + [self performMoves:@[[self moveFrom:self.canvas x:local.x - bounds[0].doubleValue / 2 + y:local.y - bounds[1].doubleValue / 2 duration:0]] expectedEnd:local]; + NSLog(@"W3C_APPLICATION_ORIGIN display=%@ app=%@ window=%@ target=%@", FBConfiguration.sharedInstance.currentDisplayId, + NSStringFromCGRect(appRect), NSStringFromCGRect(self.testedApplication.windows.firstMatch.frame), NSStringFromCGPoint(target)); + [self performMoves:@[appMove] expectedEnd:local]; + [self performMoves:@[appMove, [self moveFrom:@"pointer" x:0 y:-20 duration:200]] + expectedEnd:CGPointMake(local.x, local.y - 20)]; + NSMutableDictionary *appDrag = appMove.mutableCopy; + appDrag[@"duration"] = @200; + [self performMoves:@[[self moveFrom:self.canvas x:0 y:0 duration:0], appDrag, + [self moveFrom:@"pointer" x:0 y:-20 duration:200]] + expectedEnd:CGPointMake(local.x, local.y - 20)]; + [self performMoves:@[appMove, [self moveFrom:self.canvas x:0 y:-40 duration:200], + [self moveFrom:@"pointer" x:0 y:-20 duration:200]] + expectedEnd:CGPointMake(bounds[0].doubleValue / 2, bounds[1].doubleValue / 2 - 60)]; +} + +- (void)testW3CApplicationOrigin +{ + [self verifyW3CApplicationOrigin]; +} + +- (void)testW3CApplicationOriginAcrossOrientations +{ + XCTSkipIf(XCUIDevice.sharedDevice.fb_canAttemptSimulatedHingeAngleInjection, @"Covered by fold states"); + [self addTeardownBlock:^{ [self resetOrientation]; }]; + for (NSNumber *orientation in @[@(UIDeviceOrientationLandscapeLeft), @(UIDeviceOrientationLandscapeRight)]) { + [self resetOrientation]; + XCTAssertTrue([[XCUIDevice sharedDevice] fb_setDeviceInterfaceOrientation:orientation.integerValue]); + [self verifyW3CApplicationOrigin]; + } +} + +- (void)testW3CApplicationOriginAcrossFoldStates +{ + XCUIDevice *device = XCUIDevice.sharedDevice; + XCTSkipIf(!device.fb_canAttemptSimulatedHingeAngleInjection, @"Requires a foldable simulator"); + NSError *error = nil; + NSNumber *originalAngle = [device fb_getSimulatedHingeAngle:&error]; + XCTAssertNotNil(originalAngle); + [self addTeardownBlock:^{ if (nil != originalAngle) { [device fb_setSimulatedHingeAngle:originalAngle.doubleValue error:nil]; } }]; + for (NSNumber *angle in @[@0, @180, @90, @0]) { + [self.testedApplication terminate]; + XCTAssertTrue([device fb_setSimulatedHingeAngle:angle.doubleValue error:&error], @"%@", error); + XCTAssertTrue([[[FBRunLoopSpinner new] timeout:5] spinUntilTrue:^BOOL { + NSNumber *reading = [device fb_getSimulatedHingeAngle:nil]; + return nil != reading && fabs(reading.doubleValue - angle.doubleValue) < 1; + }]); + [self launchApplication]; + [self.testedApplication.buttons[@"coordinate-probe"] tap]; + self.canvas = self.testedApplication.otherElements[@"coordinate-canvas"]; + [self selectFixtureDisplay]; + [self verifyW3CApplicationOrigin]; + } +} + - (void)testViewportAndElementCorners { NSDictionary *geometry = [self measurement]; diff --git a/docs/multiple-displays.md b/docs/multiple-displays.md index 6cc168b061..6ce20e96a0 100644 --- a/docs/multiple-displays.md +++ b/docs/multiple-displays.md @@ -34,6 +34,14 @@ origins can be mixed within one action sequence; element offsets remain relative to the element's center. Element origins require `currentDisplayId` to match the element's display; they do not override the selected display. Key-only and pause-only sequences do not require the selected display to be available. +For an application-element origin on a secondary display, compute offsets from +its returned element rect, just as for any other element. XCTest may report that +rect with dimensions transposed relative to the application's window; WDA uses +the returned rect's center plus the requested offset as a viewport point and +applies the selected-display correction. Do not substitute the window's center +when calculating an offset for an application-element origin. This is separate +from the older-SDK compatibility issue below. Main-display application origins +retain their existing behavior. Element scrolling uses the visible area in the same coordinate space as the element frame, including on rotated main and secondary displays. WDA scrolling rejects an unavailable selected display instead of using an From 1f944b575b04e8123400550ee766a5c294a93c67 Mon Sep 17 00:00:00 2001 From: Kazuaki Matsuo Date: Wed, 7 Oct 2026 08:16:17 -0700 Subject: [PATCH 28/29] fix: preserve native gesture edge coordinates --- .../Commands/FBElementCommands.m | 6 ++- .../IntegrationTests/FBDisplayGestureTests.m | 43 +++++++++++++++++++ docs/multiple-displays.md | 7 ++- 3 files changed, 52 insertions(+), 4 deletions(-) diff --git a/WebDriverAgentLib/Commands/FBElementCommands.m b/WebDriverAgentLib/Commands/FBElementCommands.m index 744f1322ac..d5e1bd32d8 100644 --- a/WebDriverAgentLib/Commands/FBElementCommands.m +++ b/WebDriverAgentLib/Commands/FBElementCommands.m @@ -712,18 +712,20 @@ + (nullable XCUICoordinate *)gestureCoordinateWithOffset:(CGVector)offset } // Application-root coordinates are rotated using the main display on Duo. // A window coordinate keeps XCTest's display affinity and rotation intact. + // It is only a coordinate anchor, not a hit-test boundary. Preserve edge and + // outside offsets just as on the main display; XCTest handles their delivery. CGPoint point = CGPointMake(offset.dx, offset.dy); for (XCUIElement *window in [element childrenMatchingType:XCUIElementTypeWindow].allElementsBoundByIndex) { // Resolve the window before reading its display affinity: an unresolved // element may report displayID == 0. (void)window.frame; CGRect frame = window.wdFrame; - if (window.screen.displayID == screen.displayID && CGRectContainsPoint(frame, point)) { + if (window.screen.displayID == screen.displayID && !CGRectIsEmpty(frame)) { return FBCoordinateWithAnchorOffset(window, CGVectorMake(0, 0), CGVectorMake(point.x - frame.origin.x, point.y - frame.origin.y), error); } } - [[[FBErrorBuilder builder] withDescriptionFormat:@"No application window on display %lld contains the requested coordinate", screen.displayID] + [[[FBErrorBuilder builder] withDescriptionFormat:@"No non-empty application window on display %lld is available as a coordinate anchor", screen.displayID] buildError:error]; return nil; } diff --git a/WebDriverAgentTests/IntegrationTests/FBDisplayGestureTests.m b/WebDriverAgentTests/IntegrationTests/FBDisplayGestureTests.m index 4967b778f1..8c9987caaa 100644 --- a/WebDriverAgentTests/IntegrationTests/FBDisplayGestureTests.m +++ b/WebDriverAgentTests/IntegrationTests/FBDisplayGestureTests.m @@ -143,9 +143,49 @@ - (void)verifyNativeApplicationCoordinates } } +- (void)verifyNativeEdgeCoordinates +{ + NSArray *size = [self measurement][@"windowSize"]; + CGFloat width = size[0].doubleValue; + CGFloat height = size[1].doubleValue; + NSError *error = nil; + XCUICoordinate *center = [FBElementCommands gestureCoordinateWithOffset:CGVectorMake(width / 2, height / 2) + element:self.testedApplication error:&error]; + XCTAssertNotNil(center); + XCTAssertNil(error); + for (NSValue *value in @[ + [NSValue valueWithCGPoint:CGPointMake(width, height / 2)], + [NSValue valueWithCGPoint:CGPointMake(width / 2, height)], + [NSValue valueWithCGPoint:CGPointMake(width + 30, height / 2)], + [NSValue valueWithCGPoint:CGPointMake(width / 2, height + 30)], + [NSValue valueWithCGPoint:CGPointMake(-30, height / 2)], + [NSValue valueWithCGPoint:CGPointMake(width / 2, -30)], + ]) { + CGPoint point = value.CGPointValue; + error = nil; + XCUICoordinate *coordinate = [FBElementCommands gestureCoordinateWithOffset:CGVectorMake(point.x, point.y) + element:self.testedApplication error:&error]; + XCTAssertNotNil(coordinate, @"%@", error); + XCTAssertNil(error); + if (nil == coordinate) { + continue; + } + // Check extrapolation, not just acceptance: boundary points must not clamp. + CGPoint expected = [center coordinateWithOffset:CGVectorMake(point.x - width / 2, point.y - height / 2)].screenPoint; + XCTAssertEqualWithAccuracy(coordinate.screenPoint.x, expected.x, 1); + XCTAssertEqualWithAccuracy(coordinate.screenPoint.y, expected.y, 1); + } +} + +- (void)testNativeEdgeCoordinates +{ + [self verifyNativeEdgeCoordinates]; +} + - (void)testNativeApplicationCoordinates { [self verifyNativeApplicationCoordinates]; + [self verifyNativeEdgeCoordinates]; } - (void)testNativeMainDisplayDefaultCoordinates @@ -156,6 +196,7 @@ - (void)testNativeMainDisplayDefaultCoordinates [self verifyNativeApplicationCoordinates]; FBConfiguration.sharedInstance.currentDisplayId = @([FBScreen displayID]); [self verifyNativeApplicationCoordinates]; + [self verifyNativeEdgeCoordinates]; } - (void)verifyNativeCoordinatesInOrientation:(UIDeviceOrientation)orientation @@ -174,6 +215,7 @@ - (void)verifyNativeCoordinatesInOrientation:(UIDeviceOrientation)orientation && ![geometry[@"rotationInProgress"] boolValue]; }]); [self verifyNativeApplicationCoordinates]; + [self verifyNativeEdgeCoordinates]; } - (void)testNativeCoordinatesInLandscapeLeft @@ -228,6 +270,7 @@ - (void)testNativeCoordinatesAcrossFoldStates [self.testedApplication.buttons[@"coordinate-probe"] tap]; [self selectFixtureDisplay]; [self verifyNativeApplicationCoordinates]; + [self verifyNativeEdgeCoordinates]; } } diff --git a/docs/multiple-displays.md b/docs/multiple-displays.md index 6ce20e96a0..a484369d1a 100644 --- a/docs/multiple-displays.md +++ b/docs/multiple-displays.md @@ -50,8 +50,11 @@ in the same coordinate space across its drags. Keep the device orientation and fold state stable until the command completes; change the display setting between commands. Scroll-to-visible takes a fresh parent snapshot for each scroll step. -Native application-root coordinates on a secondary display require a containing -application window on that display. Requests outside those windows fail explicitly. +Native application-root coordinates on a secondary display require a +non-empty application window on that display as a coordinate anchor. Edge and +outside coordinates are preserved, as on the main display; XCTest determines +how the gesture is delivered. No hit testing or clamping is performed here. +Requests fail if no such window exists on the selected display. This gesture support builds on [#1269](https://github.com/appium/WebDriverAgent/pull/1269). From 5898cb6e84dcfc97ce195c65a088c315ddcabcc7 Mon Sep 17 00:00:00 2001 From: Kazuaki Matsuo Date: Thu, 8 Oct 2026 07:48:29 -0700 Subject: [PATCH 29/29] shorten --- .../Utilities/FBW3CActionsSynthesizer.m | 10 +-- docs/multiple-displays.md | 76 +++++-------------- 2 files changed, 20 insertions(+), 66 deletions(-) diff --git a/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m b/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m index 79dc3c6a73..5884844927 100644 --- a/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m +++ b/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m @@ -315,13 +315,9 @@ - (nullable XCUICoordinate *)positionWithError:(NSError **)error } if (nil != element) { - // Application origins are converted to viewport coordinates above and need - // its display correction. Ordinary element anchors use their own display. - // This assumes XCTest can resolve that app's coordinates correctly. On the - // Duo iOS 27.1 simulator, an SDK 26.5 compatibility app misses inner-display - // taps even in pure XCTest without WDA; the same app built with SDK 27.2 works. - // Do not apply the viewport offset as a workaround for this separate issue. - // See docs/multiple-displays.md#legacy-sdk-compatibility-windows for evidence. + // Application origins need viewport display correction; other elements use their own display. + // This correction does not fix XCTest's legacy-SDK compatibility issue: + // see docs/multiple-displays.md#legacy-sdk-compatibility-windows. self.usesDisplayCorrection = element.elementType == XCUIElementTypeApplication; if (nil == x && nil == y) { return [self hitpointWithElement:element positionOffset:nil error:error]; diff --git a/docs/multiple-displays.md b/docs/multiple-displays.md index a484369d1a..eedb6133f0 100644 --- a/docs/multiple-displays.md +++ b/docs/multiple-displays.md @@ -60,65 +60,23 @@ This gesture support builds on [#1269](https://github.com/appium/WebDriverAgent/ ## Legacy SDK compatibility windows -Older-SDK apps can run in a compatibility window on iPhone Duo. Selecting the -correct `currentDisplayId` is still required, but it does not address failures -in XCTest's handling of that compatibility layout. This is separate from sending -a gesture to the wrong display. - -A controlled comparison on the **iPhone Duo simulator, iOS 27.1 (24A94401)** -reproduced missed input without WDA. The same coordinate-probe app source was built -with **SDK 26.5 (Xcode 26.6)** and **SDK 27.2 (Xcode 27.2 beta 2)**, using separate -bundle IDs. Both were driven by the same Xcode 27.2 beta 2 UI-test runner. The -runner did not link WDA, and checked that WDA's `FBScreen` class was absent. -Only pose setup used simulated hinge injection; measured taps used public -`XCUIElement.tap()` and `coordinate(withNormalizedOffset:).tap()` APIs. - -Each configuration tested the element center and four normalized positions. -The app recorded actual canvas-local touch coordinates and a touch counter; -returning from a tap without an XCTest error did not count as successful input. -Closed and fully open poses were repeated with the app order reversed, and the -second run also covered a 90-degree pose: - -| Duo pose | SDK 26.5: touches received / attempted | SDK 27.2: touches received / attempted | -| --- | --- | --- | -| Closed (0 degrees) | 10 / 10 | 10 / 10 | -| Fully open (180 degrees) | 0 / 10 | 10 / 10 | -| Book (90 degrees, approached from fully open) | 0 / 5 | 5 / 5 | - -Successful taps differed from the expected canvas-local position by less than -0.17 points on either axis. The old-SDK app reported a 375 x 667 logical window -in both closed and open poses. On the inner display, XCTest reported the canvas -frame as approximately `(469, 6.67, 136.33, 111.67)`, while the app's own window -coordinates were `(20, 218, 335, 409)`. Different coordinate spaces can legitimately -have different frames; the evidence of failure is the missing delivered touches, -not the frame difference alone. - -This isolates the observed failure to the Apple compatibility-display, -accessibility, or XCTest path rather than WDA's gesture-coordinate calculations. -It does not identify the failing internal Apple component. Results are limited -to this simulator/runtime, fixture, and SDK pair; physical Duo behavior and other -SDK combinations remain unverified. Rebuilding the fixture with SDK 27.2 avoided -the failure in this comparison, but is not a guarantee for every app. - -A further round trip exercised `0 → 45 → 90 → 135 → 180 → 90 → 0` -degrees, both with a fresh app launch at each pose and with the same app kept -running. Each pose used three direct XCTest taps and attempted to switch to a -scrollable table. The new-SDK fixture received all 42 taps. The old-SDK fixture -received 24 of 42: taps worked at 0, 45, and 90 degrees while opening and after -returning to 0, but none arrived at 135, 180, or 90 degrees while closing. In -those failing states, the direct XCTest tap could not even select the Scroll -segment. The old-SDK fixture entered scroll mode and scrolled in all eight -remaining cases. The new-SDK fixture entered scroll mode in all 14 cases; -12 swipes changed the offset and the last two warm-run swipes were already at -the end of the table. These observations cover both cold and warm transitions; -they do not imply that a hinge angle alone determines the active display or -whether input succeeds. In particular, the two 90-degree states differed. - -WDA does not apply a speculative coordinate correction for compatibility windows. -When diagnosing a similar failure, first select the app's display explicitly, -then compare direct XCTest taps and app-recorded input before changing coordinate -transforms. For Apple's SDK-dependent layout guidance, see -[Prepare your app for iPhone Duo](https://developer.apple.com/videos/play/tech-talks/111461/). +“Legacy SDK” here means the SDK used to build the app: the tested older app was +built with **iOS SDK 26.5 (Xcode 26.6)** and ran in a compatibility window on +iPhone Duo. It does not refer to the device's iOS version or the SDK used to build +WDA. See Apple's [SDK-dependent layout guidance](https://developer.apple.com/videos/play/tech-talks/111461/). + +On the **iPhone Duo simulator, iOS 27.1 (24A94401), with an Xcode 27.2 beta 2 +UI-test runner**, direct XCTest element and coordinate taps failed to deliver +input to that app in fully open and some intermediate poses. The same app source +built with **SDK 27.2** received the taps. The failure reproduced without WDA; +it is a known limitation observed in the Apple compatibility-layout/XCTest path, +although the failing internal component has not been identified. Physical devices +and other SDK/runtime combinations remain unverified. + +Select the app's `currentDisplayId` explicitly, but be aware that selecting the +correct display does not resolve this issue. WDA does not apply a compatibility +coordinate correction. Rebuilding with SDK 27.2 avoided the failure in the tested +app; for diagnosis, compare direct XCTest taps with app-recorded input. ## XCTest recording on the Duo simulator