diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 17c1323da..0965ab9ce 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -252,7 +252,12 @@ extension RunnerTests { ) case .tap: if let selectorKey = command.selectorKey, let selectorValue = command.selectorValue { - let match = findElement(app: activeApp, selectorKey: selectorKey, selectorValue: selectorValue) + let match = findElement( + app: activeApp, + selectorKey: selectorKey, + selectorValue: selectorValue, + allowNonHittableFallback: command.allowNonHittableCoordinateFallback == true + ) if match.isAmbiguous { return Response(ok: false, error: ErrorPayload(code: "AMBIGUOUS_MATCH", message: "selector matched multiple elements")) } @@ -264,16 +269,24 @@ extension RunnerTests { var outcome = RunnerInteractionOutcome.performed let timing = measureGesture { withTemporaryScrollIdleTimeoutIfSupported(activeApp) { - outcome = activateElement(app: activeApp, element: element, action: "tap by selector") + if match.usedNonHittableFallback { + // Maestro compatibility: RN E2E backdoor controls can be 1x1 and + // reported non-hittable by XCTest, while Maestro still taps their + // resolved bounds. Keep this behind the explicit replay-only flag. + outcome = tapAt(app: activeApp, x: frame.midX, y: frame.midY) + } else { + outcome = activateElement(app: activeApp, element: element, action: "tap by selector") + } } } if let response = unsupportedResponse(for: outcome) { return response } + waitForTextEntryReadinessAfterTap(app: activeApp, element: element) return Response( ok: true, data: DataPayload( - message: "tapped", + message: match.usedNonHittableFallback ? "tapped via non-hittable coordinate fallback" : "tapped", gestureStartUptimeMs: timing.gestureStartUptimeMs, gestureEndUptimeMs: timing.gestureEndUptimeMs, x: touchFrame?.x, @@ -729,6 +742,25 @@ extension RunnerTests { dismissed: result.dismissed ) ) + case .keyboardReturn: + let result = pressKeyboardReturn(app: activeApp) + if !result.pressed { + return Response( + ok: false, + error: ErrorPayload( + code: "UNSUPPORTED_OPERATION", + message: "Unable to press the iOS keyboard return key" + ) + ) + } + return Response( + ok: true, + data: DataPayload( + message: "keyboardReturn", + visible: result.visible, + wasVisible: result.wasVisible + ) + ) case .alert: let action = (command.action ?? "get").lowercased() guard let alert = resolveAlert(app: activeApp) else { @@ -839,7 +871,27 @@ extension RunnerTests { } let delaySeconds = Double(max(command.delayMs ?? 0, 0)) / 1000.0 let textEntryMode = resolveTextEntryMode(command) - let target = focusTextInputForTextEntry(app: activeApp, x: command.x, y: command.y) + let target: TextEntryTarget + if let selectorKey = command.selectorKey, let selectorValue = command.selectorValue { + let match = findElement( + app: activeApp, + selectorKey: selectorKey, + selectorValue: selectorValue, + allowNonHittableFallback: command.allowNonHittableCoordinateFallback == true + ) + if match.isAmbiguous { + return Response(ok: false, error: ErrorPayload(code: "AMBIGUOUS_MATCH", message: "selector matched multiple elements")) + } + guard let element = match.element else { + return Response(ok: false, error: ErrorPayload(code: "NO_MATCH", message: "selector did not match an element")) + } + guard isTextEntryElement(element) else { + return Response(ok: false, error: ErrorPayload(code: "INVALID_TARGET", message: "selector did not match a text input")) + } + target = focusTextInputForTextEntry(app: activeApp, element: element) + } else { + target = focusTextInputForTextEntry(app: activeApp, x: command.x, y: command.y) + } if textEntryMode == .replacement { guard target.element != nil else { let message = @@ -867,6 +919,17 @@ extension RunnerTests { ) ) } - return Response(ok: true, data: DataPayload(message: textResult.repaired ? "typed after repair" : "typed")) + let point = target.refreshPoint + let frame = activeApp.frame + return Response( + ok: true, + data: DataPayload( + message: textResult.repaired ? "typed after repair" : "typed", + x: point.map { Double($0.x) }, + y: point.map { Double($0.y) }, + referenceWidth: frame.isEmpty ? nil : Double(frame.width), + referenceHeight: frame.isEmpty ? nil : Double(frame.height) + ) + ) } } diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index 778b81a13..5a87c01a4 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -27,6 +27,7 @@ extension RunnerTests { struct SelectorElementMatch { let element: XCUIElement? let isAmbiguous: Bool + let usedNonHittableFallback: Bool } enum TextTypingRepairMode { @@ -177,10 +178,15 @@ extension RunnerTests { return element.exists ? element : nil } - func findElement(app: XCUIApplication, selectorKey: String, selectorValue: String) -> SelectorElementMatch { + func findElement( + app: XCUIApplication, + selectorKey: String, + selectorValue: String, + allowNonHittableFallback: Bool = false + ) -> SelectorElementMatch { let value = selectorValue.trimmingCharacters(in: .whitespacesAndNewlines) guard !value.isEmpty else { - return SelectorElementMatch(element: nil, isAmbiguous: false) + return SelectorElementMatch(element: nil, isAmbiguous: false, usedNonHittableFallback: false) } let predicate: NSPredicate switch selectorKey { @@ -193,21 +199,47 @@ extension RunnerTests { case "text": predicate = NSPredicate(format: "label ==[c] %@ OR identifier ==[c] %@ OR value ==[c] %@", value, value, value) default: - return SelectorElementMatch(element: nil, isAmbiguous: false) + return SelectorElementMatch(element: nil, isAmbiguous: false, usedNonHittableFallback: false) } var matchedElement: XCUIElement? + var nonHittableElement: XCUIElement? let matches = app.descendants(matching: .any).matching(predicate).allElementsBoundByIndex for element in matches where element.exists { - guard element.isHittable else { + if !element.isHittable { + if allowNonHittableFallback && hasTappableFrame(app: app, element: element) { + guard nonHittableElement == nil else { + return SelectorElementMatch(element: nil, isAmbiguous: true, usedNonHittableFallback: false) + } + nonHittableElement = element + } continue } guard matchedElement == nil else { - return SelectorElementMatch(element: nil, isAmbiguous: true) + return SelectorElementMatch(element: nil, isAmbiguous: true, usedNonHittableFallback: false) } matchedElement = element } - return SelectorElementMatch(element: matchedElement, isAmbiguous: false) + if let matchedElement { + return SelectorElementMatch(element: matchedElement, isAmbiguous: false, usedNonHittableFallback: false) + } + return SelectorElementMatch( + element: nonHittableElement, + isAmbiguous: false, + usedNonHittableFallback: nonHittableElement != nil + ) + } + + private func hasTappableFrame(app: XCUIApplication, element: XCUIElement) -> Bool { + let frame = element.frame + if frame.isEmpty { + return false + } + let appFrame = app.frame + if appFrame.isEmpty { + return true + } + return appFrame.contains(CGPoint(x: frame.midX, y: frame.midY)) } func queryElement(app: XCUIApplication, selectorKey: String, selectorValue: String) -> Response { @@ -303,7 +335,7 @@ extension RunnerTests { switch element.elementType { case .textField, .secureTextField, .searchField, .textView: let frame = element.frame - return !frame.isEmpty && frame.contains(point) + return !frame.isEmpty && frameContainsPoint(frame, point, tolerance: 2) default: return false } @@ -334,20 +366,31 @@ extension RunnerTests { return matched } + private func frameContainsPoint(_ frame: CGRect, _ point: CGPoint, tolerance: CGFloat) -> Bool { + point.x >= frame.minX - tolerance + && point.x <= frame.maxX + tolerance + && point.y >= frame.minY - tolerance + && point.y <= frame.maxY + tolerance + } + func focusedTextInput(app: XCUIApplication) -> XCUIElement? { +#if os(iOS) + return nil +#else var focused: XCUIElement? let exceptionMessage = RunnerObjCExceptionCatcher.catchException({ - let candidate = app + let candidates = app .descendants(matching: .any) .matching(NSPredicate(format: "hasKeyboardFocus == 1")) - .firstMatch - guard candidate.exists else { return } - - switch candidate.elementType { - case .textField, .secureTextField, .searchField, .textView: - focused = candidate - default: - return + .allElementsBoundByIndex + for candidate in candidates where candidate.exists { + switch candidate.elementType { + case .textField, .secureTextField, .searchField, .textView: + focused = candidate + return + default: + continue + } } }) if let exceptionMessage { @@ -358,6 +401,7 @@ extension RunnerTests { return nil } return focused +#endif } func stabilizeTextInputBeforeTyping(app: XCUIApplication, target: XCUIElement?) -> XCUIElement? { @@ -417,6 +461,36 @@ extension RunnerTests { ) } + func focusTextInputForTextEntry(app: XCUIApplication, element: XCUIElement) -> TextEntryTarget { + let point = textEntryRefreshPoint(for: element) + if let point { + _ = tapAt(app: app, x: point.x, y: point.y) + } + let stabilized = stabilizeTextInputBeforeTyping(app: app, target: element) + let resolved = waitForTextEntryReadiness( + app: app, + target: TextEntryTarget( + element: stabilized ?? element, + refreshPoint: point, + prefersFocusedElement: false + ) + ) ?? stabilized ?? element + return TextEntryTarget( + element: resolved, + refreshPoint: textEntryRefreshPoint(for: resolved) ?? point, + prefersFocusedElement: false + ) + } + + func isTextEntryElement(_ element: XCUIElement) -> Bool { + switch element.elementType { + case .textField, .secureTextField, .searchField, .textView: + return true + default: + return false + } + } + func resolveTextEntryMode(_ command: Command) -> TextTypingRepairMode { switch command.textEntryMode { case "append": @@ -597,7 +671,7 @@ extension RunnerTests { guard let observedText = editableTextValue(for: targetElement) else { return TextEntryResult(verified: nil, repaired: repaired, expectedText: expectedText, observedText: nil) } - guard observedText == expectedText else { + guard textEntryValueMatchesExpected(targetElement, observedText: observedText, expectedText: expectedText) else { return TextEntryResult( verified: false, repaired: repaired, @@ -613,7 +687,11 @@ extension RunnerTests { return TextEntryResult(verified: nil, repaired: repaired, expectedText: expectedText, observedText: nil) } latestObservedText = nextObservedText - guard nextObservedText == expectedText else { + guard textEntryValueMatchesExpected( + resolveTextEntryElement(app: app, target: target), + observedText: nextObservedText, + expectedText: expectedText + ) else { return TextEntryResult( verified: false, repaired: repaired, @@ -630,6 +708,28 @@ extension RunnerTests { ) } + private func textEntryValueMatchesExpected( + _ element: XCUIElement?, + observedText: String, + expectedText: String + ) -> Bool { + if observedText == expectedText { + return true + } + guard hasTextEntrySubmitSuffix(expectedText), element?.elementType != .textView else { + return false + } + var submittedText = expectedText + while hasTextEntrySubmitSuffix(submittedText) { + submittedText.removeLast() + } + return observedText == submittedText + } + + private func hasTextEntrySubmitSuffix(_ text: String) -> Bool { + text.hasSuffix("\n") || text.hasSuffix("\r") + } + private func expectedTextEntryValue( typedText: String, mode: TextTypingRepairMode, @@ -661,7 +761,11 @@ extension RunnerTests { guard let observedText = editableTextValue(for: resolveTextEntryElement(app: app, target: target)) else { return false } - if observedText == expectedText { + if textEntryValueMatchesExpected( + resolveTextEntryElement(app: app, target: target), + observedText: observedText, + expectedText: expectedText + ) { return false } latestObservedText = observedText @@ -678,7 +782,11 @@ extension RunnerTests { guard let latestObservedText else { return false } - guard latestObservedText != expectedText else { + guard !textEntryValueMatchesExpected( + resolveTextEntryElement(app: app, target: target), + observedText: latestObservedText, + expectedText: expectedText + ) else { return false } return isRepairableTextEntryMismatch( @@ -780,6 +888,35 @@ extension RunnerTests { #endif } + func waitForTextEntryReadinessAfterTap(app: XCUIApplication, element: XCUIElement) { +#if os(iOS) + switch element.elementType { + case .textField, .secureTextField, .searchField, .textView: + if waitForFocusedTextInput(app: app, timeout: TextEntryTiming.readinessTimeout) != nil { + return + } + let frame = element.frame + if !frame.isEmpty { + _ = tapAt(app: app, x: frame.midX, y: frame.midY) + _ = waitForFocusedTextInput(app: app, timeout: TextEntryTiming.readinessTimeout) + } + default: + return + } +#endif + } + + private func waitForFocusedTextInput(app: XCUIApplication, timeout: TimeInterval) -> XCUIElement? { + let deadline = Date().addingTimeInterval(timeout) + while Date() < deadline { + if let focused = focusedTextInput(app: app) { + return focused + } + sleepFor(TextEntryTiming.pollInterval) + } + return focusedTextInput(app: app) + } + private func textEntryRefreshPoint(for element: XCUIElement?) -> CGPoint? { guard let element else { return nil @@ -843,6 +980,85 @@ extension RunnerTests { #endif } + func pressKeyboardReturn(app: XCUIApplication) -> (wasVisible: Bool, pressed: Bool, visible: Bool) { +#if os(tvOS) + return (wasVisible: false, pressed: pressTvRemote(.select), visible: false) +#elseif os(iOS) + let wasVisible = isKeyboardVisible(app: app) + if tapKeyboardReturnControl(app: app) { + sleepFor(0.2) + return (wasVisible: wasVisible, pressed: true, visible: isKeyboardVisible(app: app)) + } + + var typed = false + let exceptionMessage = RunnerObjCExceptionCatcher.catchException({ + app.typeText(XCUIKeyboardKey.return.rawValue) + typed = true + }) + if let exceptionMessage { + NSLog( + "AGENT_DEVICE_RUNNER_KEYBOARD_RETURN_IGNORED_EXCEPTION=%@", + exceptionMessage + ) + if let singleTarget = singleTextEntryElement(app: app) { + return pressKeyboardReturn(on: singleTarget, app: app, wasVisible: wasVisible) + } + return (wasVisible: wasVisible, pressed: false, visible: isKeyboardVisible(app: app)) + } + sleepFor(0.2) + return (wasVisible: wasVisible, pressed: typed, visible: isKeyboardVisible(app: app)) +#else + return (wasVisible: false, pressed: false, visible: false) +#endif + } + + private func pressKeyboardReturn( + on element: XCUIElement, + app: XCUIApplication, + wasVisible: Bool + ) -> (wasVisible: Bool, pressed: Bool, visible: Bool) { + let exceptionMessage = RunnerObjCExceptionCatcher.catchException({ + element.tap() + element.typeText(XCUIKeyboardKey.return.rawValue) + }) + if let exceptionMessage { + NSLog( + "AGENT_DEVICE_RUNNER_KEYBOARD_RETURN_TARGET_IGNORED_EXCEPTION=%@", + exceptionMessage + ) + return (wasVisible: wasVisible, pressed: false, visible: isKeyboardVisible(app: app)) + } + sleepFor(0.2) + return (wasVisible: wasVisible, pressed: true, visible: isKeyboardVisible(app: app)) + } + + private func singleTextEntryElement(app: XCUIApplication) -> XCUIElement? { +#if os(iOS) + var matches: [XCUIElement] = [] + let exceptionMessage = RunnerObjCExceptionCatcher.catchException({ + matches = app.descendants(matching: .any).allElementsBoundByIndex.filter { element in + guard element.exists else { return false } + switch element.elementType { + case .textField, .secureTextField, .searchField, .textView: + return true + default: + return false + } + } + }) + if let exceptionMessage { + NSLog( + "AGENT_DEVICE_RUNNER_KEYBOARD_RETURN_TEXT_ENTRY_QUERY_IGNORED_EXCEPTION=%@", + exceptionMessage + ) + return nil + } + return matches.count == 1 ? matches[0] : nil +#else + return nil +#endif + } + private func tapKeyboardDismissControl(app: XCUIApplication) -> Bool { #if os(tvOS) return false @@ -880,6 +1096,22 @@ extension RunnerTests { #endif } + private func tapKeyboardReturnControl(app: XCUIApplication) -> Bool { +#if os(iOS) + for label in ["return", "Return", "Enter", "Go", "Search", "Next", "Done", "Send", "Join"] { + let candidates = [ + app.keyboards.buttons[label], + app.keyboards.keys[label], + ] + if let hittable = candidates.first(where: { $0.exists && $0.isHittable }) { + hittable.tap() + return true + } + } +#endif + return false + } + private func isKeyboardAccessoryControl(_ element: XCUIElement, keyboardFrame: CGRect) -> Bool { let frame = element.frame guard !frame.isEmpty && !keyboardFrame.isEmpty else { @@ -942,11 +1174,24 @@ extension RunnerTests { guard !normalizedValue.isEmpty else { return false } - guard let placeholder = element.placeholderValue?.trimmingCharacters(in: .whitespacesAndNewlines), - !placeholder.isEmpty else { + let placeholder = element.placeholderValue?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + if !placeholder.isEmpty && normalizedValue == placeholder { + return true + } + if isGenericTextInputLabel(normalizedValue) { + return true + } + let normalizedLabel = element.label.trimmingCharacters(in: .whitespacesAndNewlines) + return normalizedLabel == normalizedValue && isGenericTextInputLabel(normalizedLabel) + } + + private func isGenericTextInputLabel(_ value: String) -> Bool { + switch value { + case "Text input field": + return true + default: return false } - return normalizedValue == placeholder } private func readableText(for element: XCUIElement) -> String? { diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index de5fa632f..5e60b4906 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -23,6 +23,7 @@ enum CommandType: String, Codable { case rotate case appSwitcher case keyboardDismiss + case keyboardReturn case alert case pinch case rotateGesture @@ -39,6 +40,7 @@ struct Command: Codable { let text: String? let selectorKey: String? let selectorValue: String? + let allowNonHittableCoordinateFallback: Bool? let delayMs: Int? let textEntryMode: String? let clearFirst: Bool? diff --git a/src/__tests__/client.test.ts b/src/__tests__/client.test.ts index 1ed0a2174..79abe7822 100644 --- a/src/__tests__/client.test.ts +++ b/src/__tests__/client.test.ts @@ -492,6 +492,21 @@ test('replay.run keeps deprecated maestro option as backend alias', async () => assert.equal(setup.calls[0]?.flags?.replayBackend, 'maestro'); }); +test('replay.run forwards timeout budget', async () => { + const setup = createTransport(async () => ({ ok: true, data: {} })); + const client = createAgentDeviceClient(setup.config, { transport: setup.transport }); + + await client.replay.run({ + path: './flows/mod-lists.yaml', + backend: 'maestro', + timeoutMs: 240_000, + }); + + assert.equal(setup.calls.length, 1); + assert.equal(setup.calls[0]?.command, 'replay'); + assert.equal(setup.calls[0]?.flags?.timeoutMs, 240_000); +}); + test('client.command.wait prepares selector options and rejects invalid selectors', async () => { const setup = createTransport(async () => ({ ok: true, diff --git a/src/backend.ts b/src/backend.ts index eae5c551f..9cd46589d 100644 --- a/src/backend.ts +++ b/src/backend.ts @@ -92,7 +92,7 @@ export type BackendBackOptions = { }; export type BackendKeyboardOptions = { - action: 'status' | 'get' | 'dismiss'; + action: 'status' | 'get' | 'dismiss' | 'enter' | 'return'; }; export type BackendKeyboardResult = { diff --git a/src/cli/commands/client-command.ts b/src/cli/commands/client-command.ts index 828846300..e52bddbf9 100644 --- a/src/cli/commands/client-command.ts +++ b/src/cli/commands/client-command.ts @@ -173,10 +173,19 @@ function readKeyboardAction( ): KeyboardCommandOptions['action'] | undefined { const action = value?.toLowerCase(); if (action === 'get') return 'status'; - if (action === undefined || action === 'status' || action === 'dismiss') { + if ( + action === undefined || + action === 'status' || + action === 'dismiss' || + action === 'enter' || + action === 'return' + ) { return action; } - throw new AppError('INVALID_ARGS', 'keyboard action must be status, get, or dismiss.'); + throw new AppError( + 'INVALID_ARGS', + 'keyboard action must be status, get, dismiss, enter, or return.', + ); } function readFiniteNumber(value: string | undefined, label: string): number | undefined { diff --git a/src/cli/commands/generic.ts b/src/cli/commands/generic.ts index b75902085..8fa93ad0e 100644 --- a/src/cli/commands/generic.ts +++ b/src/cli/commands/generic.ts @@ -61,6 +61,7 @@ const genericClientCommandRunners = { update: flags.replayUpdate, backend: flags.replayMaestro ? 'maestro' : undefined, env: flags.replayEnv, + timeoutMs: flags.timeoutMs, }), test: ({ client, positionals, flags }) => { announceReplayTestRun({ json: flags.json }); diff --git a/src/client-types.ts b/src/client-types.ts index 2a44be877..8d28aab89 100644 --- a/src/client-types.ts +++ b/src/client-types.ts @@ -389,7 +389,7 @@ export type RotateCommandOptions = DeviceCommandBaseOptions & { export type AppSwitcherCommandOptions = DeviceCommandBaseOptions; export type KeyboardCommandOptions = DeviceCommandBaseOptions & { - action?: 'status' | 'dismiss'; + action?: 'status' | 'dismiss' | 'enter' | 'return'; }; export type ClipboardCommandOptions = @@ -449,7 +449,7 @@ export type AppSwitcherCommandResult = CommandActionResult<'app-switcher'>; export type KeyboardCommandResult = DaemonResponseData & { platform?: 'android' | 'ios'; - action?: 'status' | 'dismiss'; + action?: 'status' | 'dismiss' | 'enter'; visible?: boolean; inputType?: string | null; inputMethodPackage?: string | null; @@ -681,6 +681,7 @@ export type ReplayRunOptions = AgentDeviceRequestOverrides & { maestro?: boolean; backend?: string; env?: string[]; + timeoutMs?: number; }; export type ReplayTestOptions = AgentDeviceRequestOverrides & diff --git a/src/commands/selector-read.ts b/src/commands/selector-read.ts index b8443d27f..0940189d6 100644 --- a/src/commands/selector-read.ts +++ b/src/commands/selector-read.ts @@ -166,13 +166,7 @@ export const findCommand: RuntimeCommand = asyn disambiguateAmbiguous: false, }); if (!resolved) { - throw new AppError('COMMAND_FAILED', formatSelectorFailure(chain, [], { unique: true })); + throw new AppError('COMMAND_FAILED', formatSelectorFailure(chain, [], { unique: true }), { + command: 'is', + reason: 'selector_not_found', + predicate: options.predicate, + selector: chain.raw, + }); } const result = evaluateIsPredicate({ predicate: options.predicate, @@ -316,6 +315,13 @@ export const isCommand: RuntimeCommand = asyn throw new AppError( 'COMMAND_FAILED', `is ${options.predicate} failed for selector ${resolved.selector.raw}: ${result.details}`, + { + command: 'is', + reason: 'predicate_failed', + predicate: options.predicate, + selector: resolved.selector.raw, + predicateDetails: result.details, + }, ); } return { @@ -400,19 +406,28 @@ async function waitForFindMatch( const timeout = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; const start = now(runtime); while (now(runtime) - start < timeout) { - const capture = await captureSelectorSnapshot(runtime, options, { - updateSession: true, - scope: shouldScopeFind(locator) ? options.query : undefined, - }); - const match = findBestMatchesByLocator(capture.snapshot.nodes, locator, options.query, { - requireRect: false, - }).matches[0]; + const { match } = await findFirstLocatorMatch(runtime, options, locator); if (match) return { kind: 'found', found: true, waitedMs: now(runtime) - start }; await sleep(runtime, POLL_INTERVAL_MS); } throw new AppError('COMMAND_FAILED', 'find wait timed out'); } +async function findFirstLocatorMatch( + runtime: AgentDeviceRuntime, + options: FindReadCommandOptions, + locator: FindLocator, +): Promise<{ capture: CapturedSnapshot; match: SnapshotNode | undefined }> { + const capture = await captureSelectorSnapshot(runtime, options, { + updateSession: true, + scope: shouldScopeFind(locator) ? options.query : undefined, + }); + const match = findBestMatchesByLocator(capture.snapshot.nodes, locator, options.query, { + requireRect: false, + }).matches[0]; + return { capture, match }; +} + async function waitForSelector( runtime: AgentDeviceRuntime, options: WaitCommandOptions, diff --git a/src/commands/system.ts b/src/commands/system.ts index adbfe7f01..a8cbc5116 100644 --- a/src/commands/system.ts +++ b/src/commands/system.ts @@ -44,7 +44,7 @@ export type SystemRotateCommandResult = { }; export type SystemKeyboardCommandOptions = CommandContext & { - action?: 'status' | 'get' | 'dismiss'; + action?: 'status' | 'get' | 'dismiss' | 'enter' | 'return'; }; export type SystemKeyboardCommandResult = @@ -60,6 +60,13 @@ export type SystemKeyboardCommandResult = state: BackendKeyboardResult; backendResult?: Record; message?: string; + } + | { + kind: 'keyboardEnterPressed'; + action: 'enter'; + state: BackendKeyboardResult; + backendResult?: Record; + message?: string; }; export type SystemClipboardCommandOptions = @@ -200,17 +207,30 @@ export const keyboardCommand: RuntimeCommand< throw new AppError('UNSUPPORTED_OPERATION', 'system.keyboard is not supported by this backend'); } const action = options.action ?? 'status'; - if (action !== 'status' && action !== 'get' && action !== 'dismiss') { - throw new AppError('INVALID_ARGS', 'system.keyboard action must be status, get, or dismiss'); + if (!isKeyboardAction(action)) { + throw new AppError( + 'INVALID_ARGS', + 'system.keyboard action must be status, get, dismiss, enter, or return', + ); } const state = await runtime.backend.setKeyboard(toBackendContext(runtime, options), { action }); const formattedBackendResult = toBackendResult(state); + const keyboardState = isKeyboardResult(state) ? state : {}; + if (action === 'enter' || action === 'return') { + return { + kind: 'keyboardEnterPressed', + action: 'enter', + state: keyboardState, + ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), + ...successText('Keyboard enter pressed'), + }; + } if (action === 'dismiss') { const dismissed = isKeyboardResult(state) ? state.dismissed : undefined; return { kind: 'keyboardDismissed', action, - state: isKeyboardResult(state) ? state : {}, + state: keyboardState, ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), ...successText(dismissed === false ? 'Keyboard already hidden' : 'Keyboard dismissed'), }; @@ -218,7 +238,7 @@ export const keyboardCommand: RuntimeCommand< return { kind: 'keyboardState', action, - state: isKeyboardResult(state) ? state : {}, + state: keyboardState, ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), }; }; @@ -348,25 +368,41 @@ function normalizeAlertResult( action: BackendAlertAction, result: BackendAlertResult, ): SystemAlertCommandResult { - if (action === 'get') { - if (result.kind !== 'alertStatus') { - throw new AppError('COMMAND_FAILED', 'system.alert get returned an invalid backend result'); - } - return { kind: 'alertStatus', action, alert: result.alert }; + switch (action) { + case 'get': + return normalizeAlertStatusResult(result); + case 'wait': + return normalizeAlertWaitResult(result); + default: + return normalizeAlertHandledResult(action, result); } - if (action === 'wait') { - if (result.kind !== 'alertWait') { - throw new AppError('COMMAND_FAILED', 'system.alert wait returned an invalid backend result'); - } - return { - kind: 'alertWait', - action, - alert: result.alert, - ...(result.waitedMs !== undefined ? { waitedMs: result.waitedMs } : {}), - ...(result.timedOut !== undefined ? { timedOut: result.timedOut } : {}), - ...successText(result.alert ? 'Alert visible' : 'Alert wait timed out'), - }; +} + +function normalizeAlertStatusResult(result: BackendAlertResult): SystemAlertCommandResult { + if (result.kind !== 'alertStatus') { + throw new AppError('COMMAND_FAILED', 'system.alert get returned an invalid backend result'); + } + return { kind: 'alertStatus', action: 'get', alert: result.alert }; +} + +function normalizeAlertWaitResult(result: BackendAlertResult): SystemAlertCommandResult { + if (result.kind !== 'alertWait') { + throw new AppError('COMMAND_FAILED', 'system.alert wait returned an invalid backend result'); } + return { + kind: 'alertWait', + action: 'wait', + alert: result.alert, + ...(result.waitedMs !== undefined ? { waitedMs: result.waitedMs } : {}), + ...(result.timedOut !== undefined ? { timedOut: result.timedOut } : {}), + ...successText(result.alert ? 'Alert visible' : 'Alert wait timed out'), + }; +} + +function normalizeAlertHandledResult( + action: Extract, + result: BackendAlertResult, +): SystemAlertCommandResult { if (result.kind !== 'alertHandled') { throw new AppError( 'COMMAND_FAILED', @@ -383,6 +419,18 @@ function normalizeAlertResult( }; } +function isKeyboardAction( + action: string, +): action is 'status' | 'get' | 'dismiss' | 'enter' | 'return' { + return ( + action === 'status' || + action === 'get' || + action === 'dismiss' || + action === 'enter' || + action === 'return' + ); +} + function isKeyboardResult(value: unknown): value is BackendKeyboardResult { return Boolean(value && typeof value === 'object'); } diff --git a/src/compat/__tests__/replay-input.test.ts b/src/compat/__tests__/replay-input.test.ts index 225b5bc56..a863f519c 100644 --- a/src/compat/__tests__/replay-input.test.ts +++ b/src/compat/__tests__/replay-input.test.ts @@ -32,7 +32,7 @@ test('parseReplayInput routes compat replay scripts through the selected parser' parsed.actions.map((action) => [action.command, action.positionals]), [ ['open', ['com.callstack.agentdevicelab']], - ['click', ['id="submit-order"']], + ['__maestroTapOn', ['id="submit-order"']], ], ); }); @@ -60,7 +60,7 @@ env: parsed.actions.map((action) => [action.command, action.positionals]), [ ['open', ['cli-app']], - ['click', ['id="shell-button"']], + ['__maestroTapOn', ['id="shell-button"']], ], ); }); diff --git a/src/compat/maestro/__tests__/replay-flow.test.ts b/src/compat/maestro/__tests__/replay-flow.test.ts index cf17308d5..30153fb6b 100644 --- a/src/compat/maestro/__tests__/replay-flow.test.ts +++ b/src/compat/maestro/__tests__/replay-flow.test.ts @@ -14,6 +14,9 @@ env: - launchApp - tapOn: id: home-open-form +- tapOn: + point: 20%,20% + label: Dismiss save password prompt - doubleTapOn: id: release-notice delay: 150 @@ -37,6 +40,11 @@ env: start: 50%, 75% end: 50%, 35% duration: 300 +- swipe: + direction: LEFT +- scrollUntilVisible: + element: Discover + direction: UP - takeScreenshot: ./screens/form.png - hideKeyboard - stopApp @@ -47,34 +55,196 @@ env: parsed.actions.map((entry) => [entry.command, entry.positionals]), [ ['open', ['com.callstack.agentdevicelab']], - ['click', ['id="home-open-form"']], + ['__maestroTapOn', ['id="home-open-form"']], + ['__maestroTapPointPercent', ['20', '20']], ['click', ['id="release-notice"']], ['click', ['label="Agent Device Tester"']], ['open', ['exp://localhost:8082']], - ['click', ['label="Full name" || text="Full name" || id="Full name"']], + ['__maestroTapOn', ['label="Full name" || text="Full name" || id="Full name"']], ['type', ['Ada Lovelace']], ['wait', ['label="Checkout form"', '5000']], - ['is', ['hidden', 'label="Missing banner"']], + ['__maestroAssertNotVisible', ['label="Missing banner"']], ['wait', ['id="submit-order"', '7000']], ['scroll', ['down']], ['scroll', ['down', '0.4']], + ['scroll', ['right']], + [ + '__maestroScrollUntilVisible', + ['label="Discover" || text="Discover" || id="Discover"', '5000', 'up'], + ], ['screenshot', ['./screens/form.png']], ['keyboard', ['dismiss']], ['close', ['com.callstack.agentdevicelab']], ], ); - assert.equal(parsed.actions[2]?.flags.doubleTap, true); - assert.equal(parsed.actions[2]?.flags.intervalMs, 150); - assert.equal(parsed.actions[3]?.flags.holdMs, 3000); + assert.equal(parsed.actions[3]?.flags.doubleTap, true); + assert.equal(parsed.actions[3]?.flags.intervalMs, 150); + assert.equal(parsed.actions[4]?.flags.holdMs, 3000); + assert.equal(parsed.actions[1]?.flags.maestro?.allowNonHittableCoordinateFallback, true); + assert.equal(parsed.actions[6]?.flags?.maestro?.allowNonHittableCoordinateFallback, undefined); +}); + +test('parseMaestroReplayFlow maps iOS openLink through the app id when available', () => { + const parsed = parseMaestroReplayFlow( + `appId: com.callstack.agentdevicelab +--- +- openLink: exp://localhost:8082 +`, + { platform: 'ios' }, + ); + + assert.deepEqual( + parsed.actions.map((entry) => [entry.command, entry.positionals]), + [['open', ['com.callstack.agentdevicelab', 'exp://localhost:8082']]], + ); +}); + +test('parseMaestroReplayFlow converts Bluesky Maestro selector compatibility syntax', () => { + const parsed = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab +--- +- eraseText +- eraseText: 12 +- tapOn: + id: likeBtn + childOf: + id: postThreadItem-by-bob.test +- tapOn: + id: postDropdownBtn + index: 0 +- tapOn: + label: Display name metadata + text: Display name +- swipe: + label: Drag feed down + from: + id: feed-drag-handle + direction: UP + duration: 350 +`); + + assert.deepEqual( + parsed.actions.map((entry) => [entry.command, entry.positionals]), + [ + ['type', ['\b'.repeat(50)]], + ['type', ['\b'.repeat(12)]], + [ + '__maestroTapOn', + ['id="likeBtn"', JSON.stringify({ childOf: 'id="postThreadItem-by-bob.test"' })], + ], + ['__maestroTapOn', ['id="postDropdownBtn"', JSON.stringify({ index: 0 })]], + ['__maestroTapOn', ['label="Display name"']], + ['__maestroSwipeOn', ['id="feed-drag-handle"', 'up', '350']], + ], + ); +}); + +test('parseMaestroReplayFlow preserves runScript as an ordered runtime action', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-maestro-runscript-')); + const scriptPath = path.join(root, 'setup.js'); + const flowPath = path.join(root, 'flow.yml'); + fs.writeFileSync(scriptPath, `output.result = SERVER_PATH`); + + const parsed = parseMaestroReplayFlow( + `appId: com.callstack.agentdevicelab +--- +- runScript: + file: ./setup.js + env: + SERVER_PATH: local +- inputText: \${output.result} +`, + { sourcePath: flowPath }, + ); + + assert.deepEqual( + parsed.actions.map((entry) => [entry.command, entry.positionals]), + [ + ['__maestroRunScript', [scriptPath]], + ['type', ['${output.result}']], + ], + ); + assert.deepEqual(parsed.actions[0]?.flags.maestro?.runScriptEnv, { SERVER_PATH: 'local' }); +}); + +test('parseMaestroReplayFlow keeps focused inputText and pressKey Enter as separate actions', () => { + const parsed = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab +--- +- inputText: hello +- pressKey: Enter +- inputText: world +`); + + assert.deepEqual( + parsed.actions.map((entry) => [entry.command, entry.positionals]), + [ + ['type', ['hello']], + ['__maestroPressEnter', []], + ['type', ['world']], + ], + ); + assert.deepEqual(parsed.actionLines, [3, 4, 5]); +}); + +test('parseMaestroReplayFlow marks tapOn before inputText for snapshot tap focus', () => { + const parsed = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab +--- +- tapOn: + id: editListNameInput +- inputText: Muted Users +`); + + assert.deepEqual( + parsed.actions.map((entry) => [entry.command, entry.positionals]), + [ + ['__maestroTapOn', ['id="editListNameInput"']], + ['type', ['Muted Users']], + ], + ); + assert.equal(parsed.actions[0]?.flags?.maestro?.allowNonHittableCoordinateFallback, undefined); +}); + +test('parseMaestroReplayFlow coalesces tapOn inputText while preserving pressKey Enter submit', () => { + const parsed = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab +--- +- tapOn: + id: e2eProxyHeaderInput +- inputText: \${output.result} +- pressKey: Enter +`); + + assert.deepEqual( + parsed.actions.map((entry) => [entry.command, entry.positionals]), + [ + ['wait', ['id="e2eProxyHeaderInput"', '30000']], + ['fill', ['id="e2eProxyHeaderInput"', '${output.result}']], + ['__maestroPressEnter', []], + ], + ); + assert.deepEqual(parsed.actionLines, [3, 3, 6]); + assert.equal(parsed.actions[1]?.flags?.maestro?.allowNonHittableCoordinateFallback, true); +}); + +test('parseMaestroReplayFlow rejects relative runScript paths without source path', () => { + assert.throws( + () => + parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab +--- +- runScript: ./setup.js +`), + (error) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + /runScript file paths/.test(error.message), + ); }); test('parseMaestroReplayFlow rejects unsupported Maestro commands', () => { assert.throws( - () => parseMaestroReplayFlow('---\n- scrollUntilVisible: Save\n'), + () => parseMaestroReplayFlow('---\n- travelThroughTime: Save\n'), (error) => error instanceof AppError && error.code === 'INVALID_ARGS' && - /scrollUntilVisible/.test(error.message) && + /travelThroughTime/.test(error.message) && /issues\/558/.test(error.message) && /issues\/new/.test(error.message) && /line 2/.test(error.message), @@ -103,52 +273,7 @@ test('parseMaestroReplayFlow preserves selector state and absolute swipe command assert.deepEqual(parsed.actionLines, [3, 6]); }); -test('parseMaestroReplayFlow maps easy Maestro device and utility commands', () => { - const parsed = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab -env: - VIDEO_PATH: ./recordings/checkout.mp4 ---- -- setAirplaneMode: true -- setAirplaneMode: false -- setLocation: - latitude: 52.2297 - longitude: 21.0122 -- setOrientation: landscapeLeft -- setPermissions: - camera: allow - microphone: deny - photos: unset - location: always -- killApp -- killApp: com.callstack.other -- pasteText: hello there -- startRecording: - path: \${VIDEO_PATH} -- stopRecording -- assertTrue: true -`); - - assert.deepEqual( - parsed.actions.map((entry) => [entry.command, entry.positionals]), - [ - ['settings', ['airplane', 'on']], - ['settings', ['airplane', 'off']], - ['settings', ['location', 'set', '52.2297', '21.0122']], - ['rotate', ['landscape-left']], - ['settings', ['permission', 'grant', 'camera']], - ['settings', ['permission', 'deny', 'microphone']], - ['settings', ['permission', 'reset', 'photos']], - ['settings', ['permission', 'grant', 'location-always']], - ['close', ['com.callstack.agentdevicelab']], - ['close', ['com.callstack.other']], - ['type', ['hello there']], - ['record', ['start', './recordings/checkout.mp4']], - ['record', ['stop']], - ], - ); -}); - -test('parseMaestroReplayFlow rejects unsupported easy-mapping variants loudly', () => { +test('parseMaestroReplayFlow rejects deferred Maestro utility commands loudly', () => { assert.throws( () => parseMaestroReplayFlow('---\n- assertTrue: "${READY}"\n'), (error) => @@ -160,11 +285,11 @@ test('parseMaestroReplayFlow rejects unsupported easy-mapping variants loudly', ); assert.throws( - () => parseMaestroReplayFlow('---\n- setPermissions:\n camera: always\n'), + () => parseMaestroReplayFlow('---\n- setPermissions:\n camera: allow\n'), (error) => error instanceof AppError && error.code === 'INVALID_ARGS' && - /setPermissions state "always"/.test(error.message) && + /setPermissions/.test(error.message) && /issues\/558/.test(error.message) && /line 2/.test(error.message), ); @@ -196,12 +321,12 @@ test('parseMaestroReplayFlow reports top-level command lines around nested lists - runFlow: commands: - tapOn: Nested -- scrollUntilVisible: Save +- travelThroughTime: Save `), (error) => error instanceof AppError && error.code === 'INVALID_ARGS' && - /scrollUntilVisible/.test(error.message) && + /travelThroughTime/.test(error.message) && /line 6/.test(error.message), ); }); @@ -251,14 +376,14 @@ onFlowComplete: assert.deepEqual( parsed.actions.map((entry) => [entry.command, entry.positionals]), [ - ['click', ['label="Before" || text="Before" || id="Before"']], - ['click', ['label="Nested" || text="Nested" || id="Nested"']], - ['click', ['id="child-repeat"']], - ['click', ['id="child-repeat"']], - ['click', ['label="iOS only" || text="iOS only" || id="iOS only"']], - ['click', ['label="Again" || text="Again" || id="Again"']], - ['click', ['label="Again" || text="Again" || id="Again"']], - ['click', ['label="After" || text="After" || id="After"']], + ['__maestroTapOn', ['label="Before" || text="Before" || id="Before"']], + ['__maestroTapOn', ['label="Nested" || text="Nested" || id="Nested"']], + ['__maestroTapOn', ['id="child-repeat"']], + ['__maestroTapOn', ['id="child-repeat"']], + ['__maestroTapOn', ['label="iOS only" || text="iOS only" || id="iOS only"']], + ['__maestroTapOn', ['label="Again" || text="Again" || id="Again"']], + ['__maestroTapOn', ['label="Again" || text="Again" || id="Again"']], + ['__maestroTapOn', ['label="After" || text="After" || id="After"']], ], ); }); @@ -279,57 +404,97 @@ test('parseMaestroReplayFlow skips platform-gated runFlow commands for other pla assert.deepEqual( parsed.actions.map((entry) => [entry.command, entry.positionals]), - [['click', ['label="Shared" || text="Shared" || id="Shared"']]], + [['__maestroTapOn', ['label="Shared" || text="Shared" || id="Shared"']]], ); }); -test('parseMaestroReplayFlow tolerates false launchApp reset options and rejects reset side effects', () => { +test('parseMaestroReplayFlow keeps visible-gated runFlow commands for runtime evaluation', () => { + const parsed = parseMaestroReplayFlow( + `appId: com.callstack.agentdevicelab +--- +- runFlow: + when: + visible: Continue + commands: + - tapOn: Continue +`, + { platform: 'ios' }, + ); + + assert.equal(parsed.actions[0]?.command, '__maestroRunFlowWhen'); + assert.deepEqual(parsed.actions[0]?.positionals, [ + 'visible', + 'label="Continue" || text="Continue" || id="Continue"', + ]); + assert.deepEqual(parsed.actions[0]?.flags.batchSteps, [ + { + command: '__maestroTapOn', + positionals: ['label="Continue" || text="Continue" || id="Continue"'], + flags: { maestro: { allowNonHittableCoordinateFallback: true } }, + }, + ]); +}); + +test('parseMaestroReplayFlow accepts launchApp reset options', () => { const parsed = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab --- - launchApp: - clearState: false - clearKeychain: false + clearState: true + arguments: + "-EXDevMenuIsOnboardingFinished": true + launchArguments: + "-Example": "ignored" stopApp: true `); assert.deepEqual( parsed.actions.map((entry) => [entry.command, entry.positionals, entry.flags]), - [['open', ['com.callstack.agentdevicelab'], { relaunch: true }]], + [ + [ + 'open', + ['com.callstack.agentdevicelab'], + { + clearAppState: true, + launchArgs: ['-EXDevMenuIsOnboardingFinished', 'true', '-Example', 'ignored'], + }, + ], + ], ); +}); +test('parseMaestroReplayFlow rejects clearKeychain instead of ignoring it', () => { assert.throws( () => parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab --- - launchApp: - clearState: true + clearKeychain: true `), (error) => error instanceof AppError && error.code === 'INVALID_ARGS' && - /clearState: true/.test(error.message) && - /line 3/.test(error.message), + /clearKeychain/.test(error.message), ); }); -test('parseMaestroReplayFlow rejects runtime-dependent flow control for now', () => { - assert.throws( - () => - parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab +test('parseMaestroReplayFlow relaunches launchApp only when clearState is absent', () => { + const withLaunchArgs = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab --- -- runFlow: - when: - visible: Continue - commands: - - tapOn: Continue -`), - (error) => - error instanceof AppError && - error.code === 'INVALID_ARGS' && - /when.visible/.test(error.message) && - /line 3/.test(error.message), - ); +- launchApp: + arguments: + "-Example": "value" +`); + const withStopApp = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab +--- +- launchApp: + stopApp: true +`); + + assert.equal(withLaunchArgs.actions[0]?.flags.relaunch, true); + assert.equal(withStopApp.actions[0]?.flags.relaunch, true); +}); +test('parseMaestroReplayFlow rejects unsupported runtime-dependent flow control', () => { assert.throws( () => parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab @@ -360,21 +525,21 @@ test('parseMaestroReplayFlow parses the test-app Maestro suite fixture', () => { parsed.actions.map((entry) => entry.command), [ 'wait', - 'click', + '__maestroTapOn', 'wait', - 'click', + '__maestroTapOn', 'type', - 'click', + '__maestroTapOn', 'type', - 'click', + '__maestroTapOn', 'wait', 'wait', 'scroll', - 'click', + '__maestroTapOn', 'wait', - 'click', + '__maestroTapOn', 'wait', - 'click', + '__maestroTapOn', 'wait', 'wait', ], diff --git a/src/compat/maestro/command-mapper.ts b/src/compat/maestro/command-mapper.ts index 76a3bc1b7..30e36d60d 100644 --- a/src/compat/maestro/command-mapper.ts +++ b/src/compat/maestro/command-mapper.ts @@ -1,23 +1,14 @@ import type { SessionAction } from '../../daemon/types.ts'; import { AppError } from '../../utils/errors.ts'; -import { - convertAssertTrue, - convertKillApp, - convertLaunchApp, - convertSetAirplaneMode, - convertSetLocation, - convertSetOrientation, - convertSetPermissions, - convertStartRecording, - convertStopApp, - convertStopRecording, -} from './device-actions.ts'; +import { convertLaunchApp, convertStopApp } from './device-actions.ts'; import { convertDoubleTapOn, + convertEraseText, convertExtendedWaitUntil, convertLongPressOn, convertPressKey, convertScroll, + convertScrollUntilVisible, convertSwipe, convertTapOn, maestroSelector, @@ -25,17 +16,15 @@ import { } from './interactions.ts'; import { action, - assertOnlyKeys, - isPlainRecord, - normalizeCommandList, - normalizePlatformValue, - readEnvMap, readTimeoutMs, + requireAppId, requireStringValue, resolveMaestroString, unsupportedCommand, - unsupportedMaestroSyntax, } from './support.ts'; +import { convertRepeat, convertRunFlow } from './flow-control.ts'; +import { convertRunScript } from './run-script.ts'; +import { MAESTRO_RUNTIME_COMMAND } from './runtime-commands.ts'; import type { MaestroCommand, MaestroCommandMapperDeps, @@ -43,7 +32,6 @@ import type { MaestroParseContext, } from './types.ts'; -const MAX_REPEAT_EXPANSIONS = 100; type MaestroCommandHandler = (params: { value: unknown; config: MaestroFlowConfig; @@ -60,39 +48,36 @@ const MAP_COMMAND_HANDLERS: Record = { inputText: ({ value, context }) => [ action('type', [resolveMaestroString(readInputText(value), context)]), ], + eraseText: ({ value }) => [convertEraseText(value)], pasteText: ({ value, context, name }) => [ action('type', [resolveMaestroString(requireStringValue(name, value), context)]), ], - openLink: ({ value, context, name }) => [ - action('open', [resolveMaestroString(requireStringValue(name, value), context)]), - ], + openLink: ({ value, config, context, name }) => [convertOpenLink(value, config, context, name)], assertVisible: ({ value, context, name }) => [ action('wait', [maestroSelector(value, name, [], context), '5000']), ], assertNotVisible: ({ value, context, name }) => [ - action('is', ['hidden', maestroSelector(value, name, [], context)]), + action(MAESTRO_RUNTIME_COMMAND.assertNotVisible, [maestroSelector(value, name, [], context)]), ], - assertTrue: ({ value, context }) => convertAssertTrue(value, context), extendedWaitUntil: ({ value, context }) => convertExtendedWaitUntil(value, context), takeScreenshot: ({ value, context, name }) => [ action('screenshot', [resolveMaestroString(requireStringValue(name, value), context)]), ], scroll: ({ value }) => [convertScroll(value)], - swipe: ({ value }) => [convertSwipe(value)], + scrollUntilVisible: ({ value, context }) => convertScrollUntilVisible(value, context), + swipe: ({ value, context }) => [convertSwipe(value, context)], hideKeyboard: () => [action('keyboard', ['dismiss'])], pressKey: ({ value }) => [convertPressKey(value)], back: () => [action('back')], - waitForAnimationToEnd: ({ value }) => [action('wait', [String(readTimeoutMs(value, 250))])], + waitForAnimationToEnd: ({ value }) => [ + action(MAESTRO_RUNTIME_COMMAND.waitForAnimationToEnd, [String(readTimeoutMs(value, 15000))]), + ], stopApp: ({ value, config, context }) => [convertStopApp(value, config, context)], - killApp: ({ value, config, context }) => [convertKillApp(value, config, context)], - setAirplaneMode: ({ value, context }) => [convertSetAirplaneMode(value, context)], - setLocation: ({ value, context }) => [convertSetLocation(value, context)], - setOrientation: ({ value, context }) => [convertSetOrientation(value, context)], - setPermissions: ({ value, context }) => convertSetPermissions(value, context), - startRecording: ({ value, context }) => [convertStartRecording(value, context)], - stopRecording: ({ value }) => [convertStopRecording(value)], - runFlow: ({ value, config, context, deps }) => convertRunFlow(value, config, context, deps), - repeat: ({ value, config, context, deps }) => convertRepeat(value, config, context, deps), + runScript: ({ value, context }) => [convertRunScript(value, context)], + runFlow: ({ value, config, context, deps }) => + convertRunFlow(value, config, context, deps, convertCommandList), + repeat: ({ value, config, context, deps }) => + convertRepeat(value, config, context, deps, convertCommandList), }; const SCALAR_COMMAND_HANDLERS: Record< @@ -102,12 +87,10 @@ const SCALAR_COMMAND_HANDLERS: Record< launchApp: (config, context) => [convertLaunchApp(undefined, config, context)], scroll: () => [action('scroll', ['down'])], hideKeyboard: () => [action('keyboard', ['dismiss'])], + eraseText: () => [convertEraseText(undefined)], back: () => [action('back')], - waitForAnimationToEnd: () => [action('wait', ['250'])], + waitForAnimationToEnd: () => [action(MAESTRO_RUNTIME_COMMAND.waitForAnimationToEnd, ['15000'])], stopApp: (config, context) => [convertStopApp(undefined, config, context)], - killApp: (config, context) => [convertKillApp(undefined, config, context)], - startRecording: () => [action('record', ['start'])], - stopRecording: () => [action('record', ['stop'])], }; export function convertMaestroCommandWithLine( @@ -156,63 +139,17 @@ function convertScalarCommand( return handler(config, context); } -function convertRunFlow( +function convertOpenLink( value: unknown, config: MaestroFlowConfig, context: MaestroParseContext, - deps: MaestroCommandMapperDeps, -): SessionAction[] { - if (typeof value === 'string') { - return deps.parseRunFlowFile(resolveMaestroString(value, context), context).actions; - } - if (!isPlainRecord(value)) { - throw new AppError('INVALID_ARGS', 'runFlow expects a file path string or map.'); - } - assertOnlyKeys(value, 'runFlow', ['file', 'commands', 'env', 'when', 'label']); - if (!shouldRunFlow(value.when, context)) return []; - - const runContext = { - ...context, - env: { ...context.env, ...readEnvMap(value.env, 'runFlow.env'), ...context.envOverrides }, - }; - if (typeof value.file === 'string') { - return deps.parseRunFlowFile(resolveMaestroString(value.file, runContext), runContext).actions; + name: string, +): SessionAction { + const url = resolveMaestroString(requireStringValue(name, value), context); + if (context.platform === 'ios' && config.appId) { + return action('open', [resolveMaestroString(requireAppId(config, name), context), url]); } - if (Array.isArray(value.commands)) { - return convertCommandList(normalizeCommandList(value.commands), config, runContext, deps); - } - throw new AppError('INVALID_ARGS', 'runFlow map requires either file or commands.'); -} - -function convertRepeat( - value: unknown, - config: MaestroFlowConfig, - context: MaestroParseContext, - deps: MaestroCommandMapperDeps, -): SessionAction[] { - if (!isPlainRecord(value)) { - throw new AppError('INVALID_ARGS', 'repeat expects a map.'); - } - assertOnlyKeys(value, 'repeat', ['times', 'commands', 'while']); - if (value.while !== undefined) { - throw unsupportedMaestroSyntax( - 'Maestro repeat.while is not supported yet. Only deterministic repeat.times is supported.', - ); - } - const times = readRepeatTimes(value.times, context); - if (!Array.isArray(value.commands)) { - throw new AppError('INVALID_ARGS', 'repeat requires a commands list.'); - } - if (times > MAX_REPEAT_EXPANSIONS) { - throw new AppError( - 'INVALID_ARGS', - `repeat.times must be <= ${MAX_REPEAT_EXPANSIONS} for deterministic replay expansion.`, - ); - } - const commands = normalizeCommandList(value.commands); - return Array.from({ length: times }).flatMap(() => - convertCommandList(commands, config, context, deps), - ); + return action('open', [url]); } function convertCommandList( @@ -225,50 +162,3 @@ function convertCommandList( convertMaestroCommandWithLine(command, config, index + 1, context, deps), ); } - -function shouldRunFlow(value: unknown, context: MaestroParseContext): boolean { - if (value === undefined || value === null) return true; - if (!isPlainRecord(value)) { - throw new AppError('INVALID_ARGS', 'runFlow.when expects a map.'); - } - assertOnlyKeys(value, 'runFlow.when', ['platform', 'visible', 'notVisible', 'true']); - rejectUnsupportedCondition(value, 'visible', 'when.visible'); - rejectUnsupportedCondition(value, 'notVisible', 'when.notVisible'); - rejectUnsupportedCondition(value, 'true', 'when.true'); - if (value.platform === undefined) return true; - const platform = normalizePlatformValue(value.platform, 'runFlow.when.platform'); - if (!context.platform) { - throw new AppError( - 'INVALID_ARGS', - 'Maestro runFlow.when.platform requires replay to be run with --platform ios|android.', - ); - } - return platform === context.platform; -} - -function readRepeatTimes(value: unknown, context: MaestroParseContext): number { - const resolved = typeof value === 'string' ? resolveMaestroString(value, context) : value; - const numeric = - typeof resolved === 'number' - ? resolved - : typeof resolved === 'string' && /^\d+$/.test(resolved) - ? Number(resolved) - : undefined; - if (numeric === undefined || !Number.isInteger(numeric) || numeric < 0) { - throw new AppError( - 'INVALID_ARGS', - 'repeat.times must be a non-negative integer or ${VAR} resolving to one.', - ); - } - return numeric; -} - -function rejectUnsupportedCondition( - value: Record, - key: string, - label: string, -): void { - if (value[key] !== undefined) { - throw unsupportedMaestroSyntax(`Maestro ${label} is not supported yet.`); - } -} diff --git a/src/compat/maestro/device-actions.ts b/src/compat/maestro/device-actions.ts index 95e33db4d..433629b0e 100644 --- a/src/compat/maestro/device-actions.ts +++ b/src/compat/maestro/device-actions.ts @@ -4,50 +4,11 @@ import { action, assertOnlyKeys, isPlainRecord, - normalizeToken, - readBooleanLiteral, requireAppId, resolveMaestroString, - resolveMaybeMaestroString, unsupportedMaestroSyntax, } from './support.ts'; -import type { MaestroFlowConfig, MaestroParseContext, PermissionCommand } from './types.ts'; - -const SUPPORTED_PERMISSION_TARGETS = new Set([ - 'accessibility', - 'calendar', - 'camera', - 'contacts', - 'contacts-limited', - 'input-monitoring', - 'location', - 'location-always', - 'media-library', - 'microphone', - 'motion', - 'notifications', - 'photos', - 'reminders', - 'screen-recording', - 'siri', -]); - -const BASIC_PERMISSION_STATES: Record = { - allow: 'grant', - grant: 'grant', - granted: 'grant', - deny: 'deny', - denied: 'deny', - reset: 'reset', - unset: 'reset', - revoke: 'reset', - revoked: 'reset', -}; - -const MODE_PERMISSION_STATES: Record = { - limited: { command: 'grant', mode: 'limited' }, - full: { command: 'grant', mode: 'full' }, -}; +import type { MaestroFlowConfig, MaestroParseContext } from './types.ts'; export function convertLaunchApp( value: unknown, @@ -70,16 +31,20 @@ export function convertLaunchApp( 'permissions', 'launchArguments', ]); - rejectTruthyLaunchOption(value, 'clearState'); - rejectTruthyLaunchOption(value, 'clearKeychain'); - rejectUnsupportedLaunchOption(value, 'arguments'); rejectUnsupportedLaunchOption(value, 'permissions'); - rejectUnsupportedLaunchOption(value, 'launchArguments'); + rejectUnsupportedLaunchOption(value, 'clearKeychain'); const appId = resolveMaestroString( typeof value.appId === 'string' ? value.appId : requireAppId(config, 'launchApp'), context, ); - return action('open', [appId], { relaunch: value.stopApp === true }); + const launchArgs = readLaunchArgs(value, context); + const shouldClearState = value.clearState === true; + const shouldRelaunch = !shouldClearState && (value.stopApp === true || launchArgs.length > 0); + return action('open', [appId], { + ...(shouldRelaunch ? { relaunch: true } : {}), + ...(shouldClearState ? { clearAppState: true } : {}), + ...(launchArgs.length > 0 ? { launchArgs } : {}), + }); } export function convertStopApp( @@ -94,173 +59,32 @@ export function convertStopApp( throw new AppError('INVALID_ARGS', 'stopApp expects a string appId or no value.'); } -export function convertSetAirplaneMode( - value: unknown, - context: MaestroParseContext, -): SessionAction { - const enabled = readBooleanLiteral(resolveMaybeMaestroString(value, context), 'setAirplaneMode'); - return action('settings', ['airplane', enabled ? 'on' : 'off']); -} - -export function convertSetLocation(value: unknown, context: MaestroParseContext): SessionAction { - if (!isPlainRecord(value)) { - throw new AppError('INVALID_ARGS', 'setLocation expects a map.'); - } - assertOnlyKeys(value, 'setLocation', ['latitude', 'longitude', 'lat', 'lon', 'lng']); - const latitude = readCoordinate(value.latitude ?? value.lat, 'setLocation.latitude', context); - const longitude = readCoordinate( - value.longitude ?? value.lon ?? value.lng, - 'setLocation.longitude', - context, - ); - return action('settings', ['location', 'set', latitude, longitude]); -} - -export function convertSetOrientation(value: unknown, context: MaestroParseContext): SessionAction { - const raw = resolveMaybeMaestroString(value, context); - if (typeof raw !== 'string') { - throw new AppError('INVALID_ARGS', 'setOrientation expects a string value.'); - } - const orientation = normalizeToken(raw); - switch (orientation) { - case 'portrait': - case 'landscape-left': - case 'landscape-right': - return action('rotate', [orientation]); - case 'portrait-upside-down': - case 'upside-down': - return action('rotate', ['portrait-upside-down']); - default: - throw unsupportedMaestroSyntax( - `Maestro setOrientation "${raw}" cannot be mapped to a supported rotate orientation.`, - ); - } +function readLaunchArgs(value: Record, context: MaestroParseContext): string[] { + return [ + ...readLaunchArgValue(value.arguments, 'launchApp.arguments', context), + ...readLaunchArgValue(value.launchArguments, 'launchApp.launchArguments', context), + ]; } -export function convertSetPermissions( - value: unknown, - context: MaestroParseContext, -): SessionAction[] { - if (!isPlainRecord(value)) { - throw new AppError('INVALID_ARGS', 'setPermissions expects a map.'); +function readLaunchArgValue(value: unknown, name: string, context: MaestroParseContext): string[] { + if (value === undefined || value === null) return []; + if (typeof value === 'string') return [resolveMaestroString(value, context)]; + if (Array.isArray(value)) { + return value.map((entry, index) => readLaunchArgScalar(entry, `${name}[${index}]`, context)); } - return Object.entries(value).map(([rawTarget, rawState]) => { - const { target, command, mode } = readPermissionMapping(rawTarget, rawState, context); - return action('settings', ['permission', command, target, ...(mode ? [mode] : [])]); - }); -} - -export function convertKillApp( - value: unknown, - config: MaestroFlowConfig, - context: MaestroParseContext, -): SessionAction { - if (value === null || value === undefined) { - return action('close', [resolveMaestroString(requireAppId(config, 'killApp'), context)]); + if (isPlainRecord(value)) { + return Object.entries(value).flatMap(([key, entry]) => [ + resolveMaestroString(key, context), + readLaunchArgScalar(entry, `${name}.${key}`, context), + ]); } - if (typeof value === 'string') return action('close', [resolveMaestroString(value, context)]); - throw new AppError('INVALID_ARGS', 'killApp expects a string appId or no value.'); + throw new AppError('INVALID_ARGS', `${name} expects a string, list, or map.`); } -export function convertStartRecording(value: unknown, context: MaestroParseContext): SessionAction { - if (value === null || value === undefined) return action('record', ['start']); - if (typeof value === 'string') - return action('record', ['start', resolveMaestroString(value, context)]); - if (!isPlainRecord(value)) { - throw new AppError('INVALID_ARGS', 'startRecording expects a string path, map, or no value.'); - } - assertOnlyKeys(value, 'startRecording', ['path', 'file']); - const rawPath = value.path ?? value.file; - if (rawPath === undefined) return action('record', ['start']); - if (typeof rawPath !== 'string') { - throw new AppError('INVALID_ARGS', 'startRecording path must be a string.'); - } - return action('record', ['start', resolveMaestroString(rawPath, context)]); -} - -export function convertStopRecording(value: unknown): SessionAction { - if (value !== null && value !== undefined) { - throw new AppError('INVALID_ARGS', 'stopRecording expects no value.'); - } - return action('record', ['stop']); -} - -export function convertAssertTrue(value: unknown, context: MaestroParseContext): SessionAction[] { - const resolved = resolveMaybeMaestroString(value, context); - if (resolved === true || (typeof resolved === 'string' && normalizeToken(resolved) === 'true')) { - return []; - } - if ( - resolved === false || - (typeof resolved === 'string' && normalizeToken(resolved) === 'false') - ) { - throw new AppError('INVALID_ARGS', 'Maestro assertTrue literal evaluated to false.'); - } - throw unsupportedMaestroSyntax('Only literal Maestro assertTrue true/false is supported.'); -} - -function readCoordinate(value: unknown, name: string, context: MaestroParseContext): string { - const resolved = resolveMaybeMaestroString(value, context); - const numeric = - typeof resolved === 'number' - ? resolved - : typeof resolved === 'string' && resolved.trim().length > 0 - ? Number(resolved) - : Number.NaN; - if (!Number.isFinite(numeric)) { - throw new AppError('INVALID_ARGS', `${name} must be a finite number.`); - } - return String(numeric); -} - -function readPermissionMapping( - rawTarget: string, - rawState: unknown, - context: MaestroParseContext, -): { target: string; command: PermissionCommand; mode?: string } { - let target = normalizeToken(rawTarget); - const resolvedState = resolveMaybeMaestroString(rawState, context); - if (typeof resolvedState !== 'string') { - throw new AppError('INVALID_ARGS', `setPermissions.${rawTarget} expects a string state.`); - } - const state = normalizeToken(resolvedState); - if (target === 'location' && state === 'always') target = 'location-always'; - - if (!SUPPORTED_PERMISSION_TARGETS.has(target)) { - throw unsupportedMaestroSyntax( - `Maestro setPermissions target "${rawTarget}" cannot be mapped to a supported settings permission target.`, - ); - } - - const basicCommand = BASIC_PERMISSION_STATES[state]; - if (basicCommand) return { target, command: basicCommand }; - - const modeMapping = MODE_PERMISSION_STATES[state]; - if (modeMapping) return { target, ...modeMapping }; - - const locationCommand = readLocationPermissionCommand(target, state); - if (locationCommand) return { target, command: locationCommand }; - - throw unsupportedMaestroSyntax( - `Maestro setPermissions state "${resolvedState}" cannot be mapped to grant, deny, or reset.`, - ); -} - -function readLocationPermissionCommand( - target: string, - state: string, -): PermissionCommand | undefined { - if (target === 'location-always' && state === 'always') return 'grant'; - if (target === 'location' && (state === 'while-in-use' || state === 'when-in-use')) { - return 'grant'; - } - return undefined; -} - -function rejectTruthyLaunchOption(value: Record, key: string): void { - if (value[key] === true) { - throw unsupportedMaestroSyntax(`Maestro launchApp ${key}: true is not supported yet.`); - } +function readLaunchArgScalar(value: unknown, name: string, context: MaestroParseContext): string { + if (typeof value === 'string') return resolveMaestroString(value, context); + if (typeof value === 'number' || typeof value === 'boolean') return String(value); + throw new AppError('INVALID_ARGS', `${name} must be a string, number, or boolean.`); } function rejectUnsupportedLaunchOption(value: Record, key: string): void { diff --git a/src/compat/maestro/flow-control.ts b/src/compat/maestro/flow-control.ts new file mode 100644 index 000000000..513d0a1cb --- /dev/null +++ b/src/compat/maestro/flow-control.ts @@ -0,0 +1,206 @@ +import type { CommandFlags } from '../../core/dispatch.ts'; +import type { SessionAction } from '../../daemon/types.ts'; +import { AppError } from '../../utils/errors.ts'; +import { maestroSelector } from './interactions.ts'; +import { MAESTRO_RUNTIME_COMMAND } from './runtime-commands.ts'; +import { + action, + assertOnlyKeys, + isPlainRecord, + normalizeCommandList, + normalizePlatformValue, + readEnvMap, + resolveMaestroString, + unsupportedMaestroSyntax, +} from './support.ts'; +import type { + MaestroCommand, + MaestroCommandMapperDeps, + MaestroFlowConfig, + MaestroParseContext, +} from './types.ts'; + +// repeat.times is expanded at parse time for deterministic replay traces. Keep +// a guardrail until repeat can execute as a runtime loop without materializing +// every child action. +const MAX_REPEAT_EXPANSIONS = 1000; + +type ConvertCommandList = ( + commands: MaestroCommand[], + config: MaestroFlowConfig, + context: MaestroParseContext, + deps: MaestroCommandMapperDeps, +) => SessionAction[]; + +export function convertRunFlow( + value: unknown, + config: MaestroFlowConfig, + context: MaestroParseContext, + deps: MaestroCommandMapperDeps, + convertCommandList: ConvertCommandList, +): SessionAction[] { + if (typeof value === 'string') { + return deps.parseRunFlowFile(resolveMaestroString(value, context), context).actions; + } + if (!isPlainRecord(value)) { + throw new AppError('INVALID_ARGS', 'runFlow expects a file path string or map.'); + } + assertOnlyKeys(value, 'runFlow', ['file', 'commands', 'env', 'when', 'label']); + const condition = readRunFlowCondition(value.when, context); + if (!condition.shouldRun) return []; + + const runContext = { + ...context, + env: { ...context.env, ...readEnvMap(value.env, 'runFlow.env'), ...context.envOverrides }, + }; + const actions = readRunFlowActions(value, config, runContext, deps, convertCommandList); + return wrapRunFlowCondition(actions, condition); +} + +export function convertRepeat( + value: unknown, + config: MaestroFlowConfig, + context: MaestroParseContext, + deps: MaestroCommandMapperDeps, + convertCommandList: ConvertCommandList, +): SessionAction[] { + if (!isPlainRecord(value)) { + throw new AppError('INVALID_ARGS', 'repeat expects a map.'); + } + assertOnlyKeys(value, 'repeat', ['times', 'commands', 'while']); + if (value.while !== undefined) { + throw unsupportedMaestroSyntax( + 'Maestro repeat.while is not supported yet. Only deterministic repeat.times is supported.', + ); + } + const times = readRepeatTimes(value.times, context); + if (!Array.isArray(value.commands)) { + throw new AppError('INVALID_ARGS', 'repeat requires a commands list.'); + } + if (times > MAX_REPEAT_EXPANSIONS) { + throw new AppError( + 'INVALID_ARGS', + `repeat.times must be <= ${MAX_REPEAT_EXPANSIONS} for deterministic replay expansion.`, + ); + } + const commands = normalizeCommandList(value.commands); + return Array.from({ length: times }).flatMap(() => + convertCommandList(commands, config, context, deps), + ); +} + +function readRunFlowActions( + value: Record, + config: MaestroFlowConfig, + context: MaestroParseContext, + deps: MaestroCommandMapperDeps, + convertCommandList: ConvertCommandList, +): SessionAction[] { + if (typeof value.file === 'string') { + return deps.parseRunFlowFile(resolveMaestroString(value.file, context), context).actions; + } + if (Array.isArray(value.commands)) { + return convertCommandList(normalizeCommandList(value.commands), config, context, deps); + } + throw new AppError('INVALID_ARGS', 'runFlow map requires either file or commands.'); +} + +type RunFlowCondition = { + shouldRun: boolean; + visibleSelector?: string; + notVisibleSelector?: string; +}; + +function readRunFlowCondition(value: unknown, context: MaestroParseContext): RunFlowCondition { + if (value === undefined || value === null) return { shouldRun: true }; + if (!isPlainRecord(value)) { + throw new AppError('INVALID_ARGS', 'runFlow.when expects a map.'); + } + assertOnlyKeys(value, 'runFlow.when', ['platform', 'visible', 'notVisible', 'true']); + rejectUnsupportedCondition(value, 'true', 'when.true'); + if (value.platform !== undefined) { + const platform = normalizePlatformValue(value.platform, 'runFlow.when.platform'); + if (!context.platform) { + throw new AppError( + 'INVALID_ARGS', + 'Maestro runFlow.when.platform requires replay to be run with --platform ios|android.', + ); + } + if (platform !== context.platform) return { shouldRun: false }; + } + return { + shouldRun: true, + ...(value.visible !== undefined + ? { visibleSelector: maestroSelector(value.visible, 'runFlow.when.visible', [], context) } + : {}), + ...(value.notVisible !== undefined + ? { + notVisibleSelector: maestroSelector( + value.notVisible, + 'runFlow.when.notVisible', + [], + context, + ), + } + : {}), + }; +} + +function wrapRunFlowCondition( + actions: SessionAction[], + condition: RunFlowCondition, +): SessionAction[] { + if (!condition.visibleSelector && !condition.notVisibleSelector) return actions; + if (condition.visibleSelector && condition.notVisibleSelector) { + throw unsupportedMaestroSyntax( + 'Maestro runFlow.when cannot combine visible and notVisible yet.', + ); + } + return [ + action( + MAESTRO_RUNTIME_COMMAND.runFlowWhen, + condition.visibleSelector + ? ['visible', condition.visibleSelector] + : ['notVisible', condition.notVisibleSelector ?? ''], + { batchSteps: actions.map(sessionActionToBatchStep) }, + ), + ]; +} + +function sessionActionToBatchStep( + entry: SessionAction, +): NonNullable[number] { + return { + command: entry.command, + positionals: entry.positionals, + flags: entry.flags, + ...(entry.runtime !== undefined ? { runtime: entry.runtime } : {}), + }; +} + +function readRepeatTimes(value: unknown, context: MaestroParseContext): number { + const resolved = typeof value === 'string' ? resolveMaestroString(value, context) : value; + const numeric = + typeof resolved === 'number' + ? resolved + : typeof resolved === 'string' && /^\d+$/.test(resolved) + ? Number(resolved) + : undefined; + if (numeric === undefined || !Number.isInteger(numeric) || numeric < 0) { + throw new AppError( + 'INVALID_ARGS', + 'repeat.times must be a non-negative integer or ${VAR} resolving to one.', + ); + } + return numeric; +} + +function rejectUnsupportedCondition( + value: Record, + key: string, + label: string, +): void { + if (value[key] !== undefined) { + throw unsupportedMaestroSyntax(`Maestro ${label} is not supported yet.`); + } +} diff --git a/src/compat/maestro/interactions.ts b/src/compat/maestro/interactions.ts index c61271606..897ec824c 100644 --- a/src/compat/maestro/interactions.ts +++ b/src/compat/maestro/interactions.ts @@ -9,19 +9,43 @@ import { resolveMaestroString, unsupportedMaestroSyntax, } from './support.ts'; +import { + parseAbsolutePoint, + parseMaestroPoint, + readScrollPositionalsFromPercentSwipe, +} from './points.ts'; +import { MAESTRO_RUNTIME_COMMAND } from './runtime-commands.ts'; import type { MaestroParseContext } from './types.ts'; +type SwipeDirection = 'up' | 'down' | 'left' | 'right'; + export function convertTapOn(value: unknown, context: MaestroParseContext): SessionAction { + if (typeof value === 'string') { + return action( + MAESTRO_RUNTIME_COMMAND.tapOn, + [visibleTextSelector(resolveMaestroString(value, context))], + maestroTapOnFlags(value), + ); + } if (isPlainRecord(value) && typeof value.point === 'string') { - assertOnlyKeys(value, 'tapOn', ['point', 'repeat', 'delay']); - const point = parsePoint(value.point); + assertOnlyKeys(value, 'tapOn', ['point', 'repeat', 'delay', 'optional', 'label']); + const point = parseMaestroPoint(value.point); + if (point.kind === 'percent') { + return action( + MAESTRO_RUNTIME_COMMAND.tapPointPercent, + [String(point.x), String(point.y)], + tapFlags(value), + ); + } return action('click', [String(point.x), String(point.y)], tapFlags(value)); } if (isPlainRecord(value)) { assertOnlyKeys(value, 'tapOn', [ 'id', 'text', + 'childOf', 'enabled', + 'index', 'selected', 'repeat', 'delay', @@ -29,17 +53,26 @@ export function convertTapOn(value: unknown, context: MaestroParseContext): Sess 'label', ]); } + const flags = maestroTapOnFlags(value); return action( - 'click', - [maestroSelector(value, 'tapOn', ['repeat', 'delay', 'optional', 'label'], context)], - tapFlags(value), + MAESTRO_RUNTIME_COMMAND.tapOn, + [ + maestroSelector( + value, + 'tapOn', + ['repeat', 'delay', 'optional', 'label', 'index', 'childOf'], + context, + ), + ...maestroTapOnRuntimeOptions(value, context), + ], + flags, ); } export function convertDoubleTapOn(value: unknown, context: MaestroParseContext): SessionAction { if (isPlainRecord(value) && typeof value.point === 'string') { assertOnlyKeys(value, 'doubleTapOn', ['point', 'delay']); - const point = parsePoint(value.point); + const point = parseAbsolutePoint(value.point); return action('click', [String(point.x), String(point.y)], doubleTapFlags(value)); } if (isPlainRecord(value)) { @@ -55,7 +88,7 @@ export function convertDoubleTapOn(value: unknown, context: MaestroParseContext) export function convertLongPressOn(value: unknown, context: MaestroParseContext): SessionAction { if (isPlainRecord(value) && typeof value.point === 'string') { assertOnlyKeys(value, 'longPressOn', ['point']); - const point = parsePoint(value.point); + const point = parseAbsolutePoint(value.point); return action('longpress', [String(point.x), String(point.y), '3000']); } if (isPlainRecord(value)) { @@ -76,6 +109,26 @@ export function readInputText(value: unknown): string { return value.text; } +export function convertEraseText(value: unknown): SessionAction { + if (value === null || value === undefined) return action('type', ['\b'.repeat(50)]); + if (typeof value === 'number' && Number.isInteger(value) && value > 0) { + return action('type', ['\b'.repeat(value)]); + } + if (!isPlainRecord(value)) { + throw new AppError('INVALID_ARGS', 'eraseText expects empty, a positive count, or a map.'); + } + assertOnlyKeys(value, 'eraseText', ['charactersToErase']); + if (value.charactersToErase === undefined) return action('type', ['\b'.repeat(50)]); + if ( + typeof value.charactersToErase !== 'number' || + !Number.isInteger(value.charactersToErase) || + value.charactersToErase <= 0 + ) { + throw new AppError('INVALID_ARGS', 'eraseText.charactersToErase must be a positive integer.'); + } + return action('type', ['\b'.repeat(value.charactersToErase)]); +} + export function convertExtendedWaitUntil( value: unknown, context: MaestroParseContext, @@ -105,20 +158,87 @@ export function convertScroll(value: unknown): SessionAction { return action('scroll', ['down']); } -export function convertSwipe(value: unknown): SessionAction { +export function convertScrollUntilVisible( + value: unknown, + context: MaestroParseContext, +): SessionAction[] { + if (typeof value === 'string') { + return [ + action(MAESTRO_RUNTIME_COMMAND.scrollUntilVisible, [ + visibleTextSelector(resolveMaestroString(value, context)), + '5000', + 'down', + ]), + ]; + } + if (!isPlainRecord(value)) { + throw new AppError('INVALID_ARGS', 'scrollUntilVisible expects a string or map.'); + } + assertOnlyKeys(value, 'scrollUntilVisible', ['element', 'direction', 'timeout']); + const selector = maestroSelector(value.element, 'scrollUntilVisible.element', [], context); + const direction = + typeof value.direction === 'string' ? readScrollUntilVisibleDirection(value.direction) : 'down'; + const timeoutMs = String(readTimeoutMs(value, 5000)); + return [action(MAESTRO_RUNTIME_COMMAND.scrollUntilVisible, [selector, timeoutMs, direction])]; +} + +export function convertSwipe(value: unknown, context: MaestroParseContext): SessionAction { if (!isPlainRecord(value)) { throw new AppError('INVALID_ARGS', 'swipe expects a map.'); } - assertOnlyKeys(value, 'swipe', ['start', 'end', 'duration']); - if (typeof value.start !== 'string' || typeof value.end !== 'string') { - throw unsupportedMaestroSyntax('Only Maestro swipe start/end coordinates are supported.'); + assertOnlyKeys(value, 'swipe', ['start', 'end', 'direction', 'duration', 'from', 'label']); + const from = value.from ?? (typeof value.label === 'string' ? value.label : undefined); + if (from !== undefined) { + return convertTargetedSwipe(value, from, context); } - const start = parseSwipePoint(value.start); - const end = parseSwipePoint(value.end); - const durationMs = - typeof value.duration === 'number' && Number.isFinite(value.duration) - ? String(Math.max(16, Math.floor(value.duration))) - : undefined; + if (typeof value.direction === 'string') { + return action('scroll', readScrollPositionalsFromDirectionSwipe(value.direction)); + } + return convertCoordinateSwipe(value); +} + +function convertTargetedSwipe( + value: Record, + from: unknown, + context: MaestroParseContext, +): SessionAction { + const direction = readSwipeDirection( + typeof value.direction === 'string' ? value.direction : 'up', + ); + return action(MAESTRO_RUNTIME_COMMAND.swipeOn, [ + maestroSelector(from, 'swipe.from', [], context), + direction, + ...swipeDurationPositionals(value), + ]); +} + +function convertCoordinateSwipe(value: Record): SessionAction { + const { start, end } = readCoordinateSwipePoints(value); + const durationMs = readSwipeDurationMs(value.duration); + return convertCoordinateSwipePoints(start, end, durationMs); +} + +function readCoordinateSwipePoints(value: Record): { + start: ReturnType; + end: ReturnType; +} { + if (typeof value.start === 'string' && typeof value.end === 'string') { + return { start: parseMaestroPoint(value.start), end: parseMaestroPoint(value.end) }; + } + throw unsupportedMaestroSyntax('Only Maestro swipe start/end coordinates are supported.'); +} + +function readSwipeDurationMs(duration: unknown): string | undefined { + return typeof duration === 'number' && Number.isFinite(duration) + ? String(Math.max(16, Math.floor(duration))) + : undefined; +} + +function convertCoordinateSwipePoints( + start: ReturnType, + end: ReturnType, + durationMs: string | undefined, +): SessionAction { if (start.kind === 'absolute' && end.kind === 'absolute') { return action('swipe', [ String(start.x), @@ -136,10 +256,50 @@ export function convertSwipe(value: unknown): SessionAction { ); } +function readScrollPositionalsFromDirectionSwipe(direction: string): string[] { + switch (readSwipeDirection(direction)) { + case 'up': + return ['down']; + case 'down': + return ['up']; + case 'left': + return ['right']; + case 'right': + return ['left']; + } +} + +function readSwipeDirection(direction: string): SwipeDirection { + const normalized = direction.toLowerCase(); + switch (normalized) { + case 'up': + case 'down': + case 'left': + case 'right': + return normalized; + default: + throw unsupportedMaestroSyntax('Maestro swipe direction must be UP, DOWN, LEFT, or RIGHT.'); + } +} + +function readScrollUntilVisibleDirection(direction: string): string { + switch (direction.toLowerCase()) { + case 'up': + case 'down': + case 'left': + case 'right': + return direction.toLowerCase(); + default: + throw unsupportedMaestroSyntax( + 'Maestro scrollUntilVisible.direction must be UP, DOWN, LEFT, or RIGHT.', + ); + } +} + export function convertPressKey(value: unknown): SessionAction { const key = requireStringValue('pressKey', value).toLowerCase(); if (key === 'back') return action('back'); - if (key === 'enter' || key === 'return') return action('press', ['return']); + if (key === 'enter' || key === 'return') return action(MAESTRO_RUNTIME_COMMAND.pressEnter); if (key === 'home') return action('home'); throw unsupportedMaestroSyntax(`Maestro pressKey "${key}" is not supported yet.`); } @@ -161,6 +321,8 @@ export function maestroSelector( terms.push(selectorTerm('id', resolveMaestroString(value.id, context))); if (typeof value.text === 'string') terms.push(selectorTerm('label', resolveMaestroString(value.text, context))); + if (typeof value.label === 'string' && terms.length === 0) + terms.push(selectorTerm('label', resolveMaestroString(value.label, context))); if (typeof value.enabled === 'boolean') terms.push(selectorTerm('enabled', String(value.enabled))); if (typeof value.selected === 'boolean') @@ -168,7 +330,7 @@ export function maestroSelector( if (terms.length === 0) { throw new AppError( 'INVALID_ARGS', - `${command} selector map must include one of id, text, enabled, or selected.`, + `${command} selector map must include one of id, text, label, enabled, or selected.`, ); } return terms.join(' '); @@ -182,6 +344,27 @@ function visibleTextSelector(value: string): string { ].join(' || '); } +function maestroTapOnRuntimeOptions(value: unknown, context: MaestroParseContext): string[] { + if (!isPlainRecord(value)) return []; + const options: { index?: number; childOf?: string } = {}; + if (value.index !== undefined) { + if (typeof value.index !== 'number' || !Number.isInteger(value.index) || value.index < 0) { + throw new AppError('INVALID_ARGS', 'tapOn.index must be a non-negative integer.'); + } + options.index = value.index; + } + if (value.childOf !== undefined) { + options.childOf = maestroSelector(value.childOf, 'tapOn.childOf', [], context); + } + return Object.keys(options).length > 0 ? [JSON.stringify(options)] : []; +} + +function swipeDurationPositionals(value: Record): string[] { + return typeof value.duration === 'number' && Number.isFinite(value.duration) + ? [String(Math.max(16, Math.floor(value.duration)))] + : []; +} + function selectorTerm(key: string, value: string): string { return `${key}=${JSON.stringify(value)}`; } @@ -189,15 +372,25 @@ function selectorTerm(key: string, value: string): string { function tapFlags(value: unknown): SessionAction['flags'] | undefined { if (!isPlainRecord(value)) return undefined; const flags: SessionAction['flags'] = {}; - if (typeof value.repeat === 'number' && Number.isInteger(value.repeat) && value.repeat > 1) { - flags.count = value.repeat; - } - if (typeof value.delay === 'number' && Number.isInteger(value.delay) && value.delay >= 0) { - flags.intervalMs = value.delay; - } + const repeat = positiveInteger(value.repeat); + const delay = nonNegativeInteger(value.delay); + if (repeat && repeat > 1) flags.count = repeat; + if (delay !== undefined) flags.intervalMs = delay; + if (value.optional === true) flags.maestro = { optional: true }; return Object.keys(flags).length > 0 ? flags : undefined; } +function maestroTapOnFlags(value: unknown): SessionAction['flags'] { + const flags = tapFlags(value) ?? {}; + return { + ...flags, + maestro: { + ...(flags.maestro ?? {}), + allowNonHittableCoordinateFallback: true, + }, + }; +} + function doubleTapFlags(value: unknown): SessionAction['flags'] { const flags: SessionAction['flags'] = { doubleTap: true }; if (isPlainRecord(value) && typeof value.delay === 'number' && Number.isInteger(value.delay)) { @@ -206,57 +399,10 @@ function doubleTapFlags(value: unknown): SessionAction['flags'] { return flags; } -function parsePoint(value: string): { x: number; y: number } { - const match = value.match(/^(\d+),(\d+)$/); - if (!match) { - throw unsupportedMaestroSyntax( - 'Only absolute Maestro point selectors like "100,200" are supported.', - ); - } - return { x: Number(match[1]), y: Number(match[2]) }; -} - -type SwipePoint = - | { - kind: 'absolute'; - x: number; - y: number; - } - | { - kind: 'percent'; - x: number; - y: number; - }; - -function parseSwipePoint(value: string): SwipePoint { - const absolute = value.match(/^\s*(\d+)\s*,\s*(\d+)\s*$/); - if (absolute) { - return { kind: 'absolute', x: Number(absolute[1]), y: Number(absolute[2]) }; - } - const percent = value.match(/^\s*(\d+(?:\.\d+)?)%\s*,\s*(\d+(?:\.\d+)?)%\s*$/); - if (percent) { - return { kind: 'percent', x: Number(percent[1]), y: Number(percent[2]) }; - } - throw unsupportedMaestroSyntax( - 'Only Maestro swipe coordinates like "100,200" or "50%,75%" are supported.', - ); -} - -function readScrollPositionalsFromPercentSwipe( - start: Extract, - end: Extract, -): string[] { - const deltaX = end.x - start.x; - const deltaY = end.y - start.y; - if (Math.abs(deltaX) === 0 && Math.abs(deltaY) === 0) { - throw new AppError('INVALID_ARGS', 'swipe start and end cannot be the same point.'); - } - const vertical = Math.abs(deltaY) >= Math.abs(deltaX); - const direction = vertical ? (deltaY < 0 ? 'down' : 'up') : deltaX < 0 ? 'right' : 'left'; - const amount = Math.min(1, Math.max(0.01, Math.abs(vertical ? deltaY : deltaX) / 100)); - return [direction, formatAmount(amount)]; +function positiveInteger(value: unknown): number | undefined { + return typeof value === 'number' && Number.isInteger(value) && value > 0 ? value : undefined; } -function formatAmount(value: number): string { - return value.toFixed(2).replace(/0+$/, '').replace(/\.$/, ''); +function nonNegativeInteger(value: unknown): number | undefined { + return typeof value === 'number' && Number.isInteger(value) && value >= 0 ? value : undefined; } diff --git a/src/compat/maestro/points.ts b/src/compat/maestro/points.ts new file mode 100644 index 000000000..c0e25ffb4 --- /dev/null +++ b/src/compat/maestro/points.ts @@ -0,0 +1,57 @@ +import { AppError } from '../../utils/errors.ts'; +import { unsupportedMaestroSyntax } from './support.ts'; + +export type MaestroPoint = + | { + kind: 'absolute'; + x: number; + y: number; + } + | { + kind: 'percent'; + x: number; + y: number; + }; + +export function parseAbsolutePoint(value: string): { x: number; y: number } { + const match = value.match(/^(\d+),(\d+)$/); + if (!match) { + throw unsupportedMaestroSyntax( + 'Only absolute Maestro point selectors like "100,200" are supported.', + ); + } + return { x: Number(match[1]), y: Number(match[2]) }; +} + +export function parseMaestroPoint(value: string): MaestroPoint { + const absolute = value.match(/^\s*(\d+)\s*,\s*(\d+)\s*$/); + if (absolute) { + return { kind: 'absolute', x: Number(absolute[1]), y: Number(absolute[2]) }; + } + const percent = value.match(/^\s*(\d+(?:\.\d+)?)%\s*,\s*(\d+(?:\.\d+)?)%\s*$/); + if (percent) { + return { kind: 'percent', x: Number(percent[1]), y: Number(percent[2]) }; + } + throw unsupportedMaestroSyntax( + 'Only Maestro swipe coordinates like "100,200" or "50%,75%" are supported.', + ); +} + +export function readScrollPositionalsFromPercentSwipe( + start: Extract, + end: Extract, +): string[] { + const deltaX = end.x - start.x; + const deltaY = end.y - start.y; + if (Math.abs(deltaX) === 0 && Math.abs(deltaY) === 0) { + throw new AppError('INVALID_ARGS', 'swipe start and end cannot be the same point.'); + } + const vertical = Math.abs(deltaY) >= Math.abs(deltaX); + const direction = vertical ? (deltaY < 0 ? 'down' : 'up') : deltaX < 0 ? 'right' : 'left'; + const amount = Math.min(1, Math.max(0.01, Math.abs(vertical ? deltaY : deltaX) / 100)); + return [direction, formatAmount(amount)]; +} + +function formatAmount(value: number): string { + return value.toFixed(2).replace(/0+$/, '').replace(/\.$/, ''); +} diff --git a/src/compat/maestro/replay-flow.ts b/src/compat/maestro/replay-flow.ts index e6b28cdaa..eab80c650 100644 --- a/src/compat/maestro/replay-flow.ts +++ b/src/compat/maestro/replay-flow.ts @@ -4,6 +4,7 @@ import { parseAllDocuments } from 'yaml'; import type { SessionAction } from '../../daemon/types.ts'; import { AppError } from '../../utils/errors.ts'; import { convertMaestroCommandWithLine } from './command-mapper.ts'; +import { MAESTRO_RUNTIME_COMMAND } from './runtime-commands.ts'; import { isPlainRecord, normalizeCommandList, normalizePlatform, readEnvMap } from './support.ts'; import type { MaestroCommand, @@ -74,7 +75,91 @@ function convertRootCommands(params: { actions.push(...converted); converted.forEach(() => actionLines.push(line)); } - return { actions, actionLines }; + return optimizeInputTextActions(actions, actionLines); +} + +function optimizeInputTextActions( + actions: SessionAction[], + actionLines: number[], +): { actions: SessionAction[]; actionLines: number[] } { + const mergedActions: SessionAction[] = []; + const mergedLines: number[] = []; + for (let index = 0; index < actions.length; index += 1) { + const action = actions[index]; + const optimized = optimizeTypedAfterTap(actions, actionLines, index); + if (optimized) { + mergedActions.push(...optimized.actions); + mergedLines.push(...optimized.actionLines); + index += optimized.consumed - 1; + continue; + } + mergedActions.push(action); + mergedLines.push(actionLines[index] ?? 1); + } + return { actions: mergedActions, actionLines: mergedLines }; +} + +function optimizeTypedAfterTap( + actions: SessionAction[], + actionLines: number[], + index: number, +): { actions: SessionAction[]; actionLines: number[]; consumed: number } | null { + const action = actions[index]; + const nextAction = actions[index + 1]; + const typedAfterTap = readPlainTypeText(nextAction); + const tapSelector = readPlainMaestroTapSelector(action); + if (typedAfterTap === null || tapSelector === null) return null; + const line = actionLines[index] ?? 1; + if (actions[index + 2]?.command !== MAESTRO_RUNTIME_COMMAND.pressEnter) { + return { actions: [clearMaestroNonHittableTap(action)], actionLines: [line], consumed: 1 }; + } + return { + actions: [ + { + ...action, + command: 'wait', + positionals: [tapSelector, '30000'], + }, + { + ...nextAction, + command: 'fill', + positionals: [tapSelector, typedAfterTap], + flags: action.flags, + }, + actions[index + 2] as SessionAction, + ], + actionLines: [line, line, actionLines[index + 2] ?? line], + consumed: 3, + }; +} + +function clearMaestroNonHittableTap(action: SessionAction): SessionAction { + const maestro = { ...(action.flags?.maestro ?? {}) }; + delete maestro.allowNonHittableCoordinateFallback; + return { + ...action, + flags: { + ...(action.flags ?? {}), + maestro: { + ...maestro, + }, + }, + }; +} + +function readPlainMaestroTapSelector(action: SessionAction | undefined): string | null { + if (action?.command !== MAESTRO_RUNTIME_COMMAND.tapOn) return null; + const [selector, ...rest] = action.positionals ?? []; + if (rest.length > 0 || typeof selector !== 'string') return null; + return selector; +} + +function readPlainTypeText(action: SessionAction | undefined): string | null { + if (action?.command !== 'type') return null; + if (action.flags && Object.keys(action.flags).length > 0) return null; + const [text, ...rest] = action.positionals ?? []; + if (rest.length > 0 || typeof text !== 'string') return null; + return text; } function parseYamlDocuments(script: string): unknown[] { diff --git a/src/compat/maestro/run-script.ts b/src/compat/maestro/run-script.ts new file mode 100644 index 000000000..49f9f242e --- /dev/null +++ b/src/compat/maestro/run-script.ts @@ -0,0 +1,200 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import vm from 'node:vm'; +import type { SessionAction } from '../../daemon/types.ts'; +import { AppError } from '../../utils/errors.ts'; +import { runCmdSync } from '../../utils/exec.ts'; +import { MAESTRO_RUNTIME_COMMAND } from './runtime-commands.ts'; +import { + action, + assertOnlyKeys, + isPlainRecord, + readEnvMap, + requireStringValue, + resolveMaestroString, +} from './support.ts'; +import type { MaestroParseContext } from './types.ts'; + +const RUN_SCRIPT_TIMEOUT_MS = 30_000; + +type HttpResponse = { + status: number; + body: string; + headers: Record; +}; + +const HTTP_REQUEST_SCRIPT = ` +const fs = require('node:fs'); +const input = JSON.parse(fs.readFileSync(0, 'utf8')); +if (typeof fetch !== 'function') { + console.error('global fetch is required for Maestro runScript http helpers'); + process.exit(1); +} +fetch(input.url, { + method: input.method, + headers: input.headers, + body: input.body, +}).then(async response => { + process.stdout.write(JSON.stringify({ + status: response.status, + body: await response.text(), + headers: Object.fromEntries(response.headers.entries()), + })); +}).catch(error => { + console.error(error && error.stack ? error.stack : String(error)); + process.exit(1); +}); +`; + +export function convertRunScript(value: unknown, context: MaestroParseContext): SessionAction { + const scriptConfig = readRunScriptConfig(value, context); + const scriptPath = resolveRunScriptPath(scriptConfig.file, context); + return action(MAESTRO_RUNTIME_COMMAND.runScript, [scriptPath], { + ...(Object.keys(scriptConfig.env).length > 0 + ? { maestro: { runScriptEnv: scriptConfig.env } } + : {}), + }); +} + +export function executeRunScriptFile(params: { + scriptPath: string; + env: Record; +}): Record { + const { scriptPath, env } = params; + const script = fs.readFileSync(scriptPath, 'utf8'); + const output: Record = {}; + + try { + // Compatibility note: node:vm is not a security sandbox. Maestro runScript + // files are trusted flow-local setup code; the timeout only bounds + // synchronous script execution. Async http.post work is bounded separately + // by the child process timeout in runHttpRequestSync. + vm.runInNewContext(script, buildScriptGlobals(env, output), { + filename: scriptPath, + timeout: RUN_SCRIPT_TIMEOUT_MS, + }); + } catch (error) { + throw new AppError( + 'COMMAND_FAILED', + `Maestro runScript failed for ${scriptPath}: ${error instanceof Error ? error.message : String(error)}`, + { scriptPath }, + error instanceof Error ? error : undefined, + ); + } + + validateOutputKeys(output, scriptPath); + return Object.fromEntries( + Object.entries(output).map(([key, rawValue]) => [ + `output.${key}`, + stringifyOutputValue(rawValue), + ]), + ); +} + +function readRunScriptConfig( + value: unknown, + context: MaestroParseContext, +): { file: string; env: Record } { + if (typeof value === 'string') { + return { file: resolveMaestroString(value, context), env: {} }; + } + if (!isPlainRecord(value)) { + throw new AppError('INVALID_ARGS', 'runScript expects a file path string or map.'); + } + assertOnlyKeys(value, 'runScript', ['file', 'env']); + const file = resolveMaestroString(requireStringValue('runScript.file', value.file), context); + const rawEnv = readEnvMap(value.env, 'runScript.env'); + const env = Object.fromEntries( + Object.entries(rawEnv).map(([key, envValue]) => [key, resolveMaestroString(envValue, context)]), + ); + return { file, env }; +} + +function resolveRunScriptPath(filePath: string, context: MaestroParseContext): string { + if (path.isAbsolute(filePath)) return filePath; + if (!context.baseDir) { + throw new AppError( + 'INVALID_ARGS', + 'runScript file paths require replay input to have a source path.', + ); + } + return path.resolve(context.baseDir, filePath); +} + +function buildScriptGlobals( + env: Record, + output: Record, +): vm.Context { + return { + ...env, + output, + json: (value: string) => JSON.parse(value) as unknown, + http: { + post: (url: string, options?: { headers?: Record; body?: string }) => + runHttpRequestSync('POST', url, options), + }, + }; +} + +function runHttpRequestSync( + method: string, + url: string, + options?: { headers?: Record; body?: string }, +): HttpResponse { + // Keep http.post synchronous from the flow author's point of view while the + // network request remains timeout-bounded independently from node:vm. + const result = runCmdSync(process.execPath, ['-e', HTTP_REQUEST_SCRIPT], { + stdin: JSON.stringify({ + method, + url, + headers: options?.headers ?? {}, + body: options?.body ?? '', + }), + timeoutMs: RUN_SCRIPT_TIMEOUT_MS, + allowFailure: true, + }); + if (result.exitCode !== 0) { + throw new AppError( + 'COMMAND_FAILED', + `Maestro runScript http.${method.toLowerCase()} failed for ${url}: ${trimHttpErrorOutput(result.stderr)}`, + { + exitCode: result.exitCode, + stderr: result.stderr, + }, + ); + } + try { + return JSON.parse(result.stdout) as HttpResponse; + } catch (error) { + throw new AppError( + 'COMMAND_FAILED', + `Maestro runScript http.${method.toLowerCase()} returned invalid JSON for ${url}`, + { + stdout: result.stdout.slice(0, 1000), + stderr: result.stderr.slice(0, 1000), + }, + error instanceof Error ? error : undefined, + ); + } +} + +function validateOutputKeys(output: Record, scriptPath: string): void { + for (const key of Object.keys(output)) { + if (!key.includes('.')) continue; + throw new AppError('INVALID_ARGS', `Maestro runScript output key cannot contain ".": ${key}`, { + scriptPath, + key, + }); + } +} + +function stringifyOutputValue(value: unknown): string { + if (typeof value === 'string') return value; + if (typeof value === 'number' || typeof value === 'boolean') return String(value); + return JSON.stringify(value); +} + +function trimHttpErrorOutput(stderr: string): string { + const trimmed = stderr.trim(); + return trimmed.length > 0 ? trimmed.slice(0, 1000) : 'request process exited without stderr'; +} diff --git a/src/compat/maestro/runtime-commands.ts b/src/compat/maestro/runtime-commands.ts new file mode 100644 index 000000000..8e354f362 --- /dev/null +++ b/src/compat/maestro/runtime-commands.ts @@ -0,0 +1,11 @@ +export const MAESTRO_RUNTIME_COMMAND = { + runFlowWhen: '__maestroRunFlowWhen', + runScript: '__maestroRunScript', + assertNotVisible: '__maestroAssertNotVisible', + pressEnter: '__maestroPressEnter', + waitForAnimationToEnd: '__maestroWaitForAnimationToEnd', + scrollUntilVisible: '__maestroScrollUntilVisible', + swipeOn: '__maestroSwipeOn', + tapOn: '__maestroTapOn', + tapPointPercent: '__maestroTapPointPercent', +} as const; diff --git a/src/compat/maestro/support.ts b/src/compat/maestro/support.ts index 3bd998faf..997a5c70d 100644 --- a/src/compat/maestro/support.ts +++ b/src/compat/maestro/support.ts @@ -63,24 +63,6 @@ export function normalizePlatformValue(value: unknown, name: string): 'android' return platform; } -export function normalizeToken(value: string): string { - return value - .trim() - .replace(/([a-z0-9])([A-Z])/g, '$1-$2') - .replace(/[\s_]+/g, '-') - .toLowerCase(); -} - -export function readBooleanLiteral(value: unknown, command: string): boolean { - if (typeof value === 'boolean') return value; - if (typeof value === 'string') { - const normalized = normalizeToken(value); - if (normalized === 'true') return true; - if (normalized === 'false') return false; - } - throw new AppError('INVALID_ARGS', `${command} expects a boolean value.`); -} - export function readEnvMap(value: unknown, name: string): Record { if (value === undefined || value === null) return {}; if (!isPlainRecord(value)) { @@ -113,15 +95,11 @@ export function requireStringValue(command: string, value: unknown): string { } export function resolveMaestroString(value: string, context: MaestroParseContext): string { - return value.replace(/\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g, (match, key: string) => { + return value.replace(/\$\{([A-Za-z_][A-Za-z0-9_.]*)\}/g, (match, key: string) => { return Object.prototype.hasOwnProperty.call(context.env, key) ? context.env[key] : match; }); } -export function resolveMaybeMaestroString(value: unknown, context: MaestroParseContext): unknown { - return typeof value === 'string' ? resolveMaestroString(value, context) : value; -} - export function unsupportedCommand(command: string): never { throw unsupportedMaestroSyntax(`Maestro command "${command}" is not supported yet.`); } diff --git a/src/compat/maestro/types.ts b/src/compat/maestro/types.ts index 81012d5e7..8ea39be92 100644 --- a/src/compat/maestro/types.ts +++ b/src/compat/maestro/types.ts @@ -31,5 +31,3 @@ export type MaestroParseContext = { export type MaestroCommandMapperDeps = { parseRunFlowFile(filePath: string, context: MaestroParseContext): MaestroReplayFlow; }; - -export type PermissionCommand = 'grant' | 'deny' | 'reset'; diff --git a/src/core/__tests__/dispatch-keyboard.test.ts b/src/core/__tests__/dispatch-keyboard.test.ts new file mode 100644 index 000000000..8cd0a8c2e --- /dev/null +++ b/src/core/__tests__/dispatch-keyboard.test.ts @@ -0,0 +1,48 @@ +import { beforeEach, test, vi } from 'vitest'; +import assert from 'node:assert/strict'; +import { promises as fs } from 'node:fs'; + +vi.mock('../../platforms/ios/runner-client.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, runIosRunnerCommand: vi.fn() }; +}); + +import { dispatchCommand } from '../dispatch.ts'; +import { runIosRunnerCommand } from '../../platforms/ios/runner-client.ts'; +import { ANDROID_EMULATOR, IOS_DEVICE } from '../../__tests__/test-utils/device-fixtures.ts'; +import { withMockedAdb } from '../../__tests__/test-utils/mocked-binaries.ts'; + +const mockRunIosRunnerCommand = vi.mocked(runIosRunnerCommand); + +beforeEach(() => { + vi.resetAllMocks(); + mockRunIosRunnerCommand.mockResolvedValue({ + message: 'keyboardReturn', + wasVisible: true, + visible: false, + }); +}); + +test('dispatch keyboard enter sends Android ENTER keyevent', async () => { + await withMockedAdb('agent-device-dispatch-keyboard-enter-', async (argsLogPath) => { + const result = await dispatchCommand(ANDROID_EMULATOR, 'keyboard', ['enter']); + + assert.equal(result?.action, 'enter'); + const logged = await fs.readFile(argsLogPath, 'utf8'); + assert.match(logged, /shell\ninput\nkeyevent\nENTER/); + }); +}); + +test('dispatch keyboard enter sends native iOS keyboard return command', async () => { + const result = await dispatchCommand(IOS_DEVICE, 'keyboard', ['return'], undefined, { + appBundleId: 'com.example.app', + }); + + assert.equal(result?.action, 'enter'); + assert.equal(result?.wasVisible, true); + assert.equal(mockRunIosRunnerCommand.mock.calls.length, 1); + assert.deepEqual(mockRunIosRunnerCommand.mock.calls[0]?.[1], { + command: 'keyboardReturn', + appBundleId: 'com.example.app', + }); +}); diff --git a/src/core/__tests__/dispatch-open.test.ts b/src/core/__tests__/dispatch-open.test.ts index c55cbbe68..1cf399a73 100644 --- a/src/core/__tests__/dispatch-open.test.ts +++ b/src/core/__tests__/dispatch-open.test.ts @@ -1,8 +1,30 @@ -import { test } from 'vitest'; +import { beforeEach, test, vi } from 'vitest'; import assert from 'node:assert/strict'; import { dispatchCommand } from '../dispatch.ts'; import { AppError } from '../../utils/errors.ts'; import type { DeviceInfo } from '../../utils/device.ts'; +import { clearIosSimulatorAppState, openIosApp } from '../../platforms/ios/apps.ts'; +import { IOS_SIMULATOR } from '../../__tests__/test-utils/device-fixtures.ts'; + +vi.mock('../../platforms/ios/apps.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + clearIosSimulatorAppState: vi.fn(async () => ({ + bundleId: 'com.example.app', + containerPath: '/tmp/com.example.app', + })), + openIosApp: vi.fn(async () => {}), + }; +}); + +const mockClearIosSimulatorAppState = vi.mocked(clearIosSimulatorAppState); +const mockOpenIosApp = vi.mocked(openIosApp); + +beforeEach(() => { + mockClearIosSimulatorAppState.mockClear(); + mockOpenIosApp.mockClear(); +}); test('dispatch open rejects URL as first argument when second URL is provided', async () => { const device: DeviceInfo = { @@ -23,3 +45,47 @@ test('dispatch open rejects URL as first argument when second URL is provided', }, ); }); + +test('dispatch open rejects Android launch arguments instead of dropping them', async () => { + const device: DeviceInfo = { + platform: 'android', + id: 'emulator-5554', + name: 'Pixel', + kind: 'emulator', + booted: true, + }; + + await assert.rejects( + () => + dispatchCommand(device, 'open', ['com.example.app'], undefined, { + launchArgs: ['--fixture', 'demo'], + }), + (error: unknown) => { + assert.equal(error instanceof AppError, true); + assert.equal((error as AppError).code, 'UNSUPPORTED_OPERATION'); + assert.match((error as AppError).message, /Apple platforms/i); + return true; + }, + ); +}); + +test('dispatch open clears Maestro iOS simulator state and launches once', async () => { + const result = await dispatchCommand(IOS_SIMULATOR, 'open', ['com.example.app'], undefined, { + clearAppState: true, + launchArgs: ['-EXDevMenuIsOnboardingFinished', 'true'], + }); + + assert.equal(result?.app, 'com.example.app'); + assert.equal(mockClearIosSimulatorAppState.mock.calls.length, 1); + assert.deepEqual(mockClearIosSimulatorAppState.mock.calls[0]?.slice(0, 2), [ + IOS_SIMULATOR, + 'com.example.app', + ]); + assert.equal(mockOpenIosApp.mock.calls.length, 1); + assert.equal(mockOpenIosApp.mock.calls[0]?.[0], IOS_SIMULATOR); + assert.equal(mockOpenIosApp.mock.calls[0]?.[1], 'com.example.app'); + assert.deepEqual(mockOpenIosApp.mock.calls[0]?.[2]?.launchArgs, [ + '-EXDevMenuIsOnboardingFinished', + 'true', + ]); +}); diff --git a/src/core/capabilities.ts b/src/core/capabilities.ts index b96573010..8ff7bfeb7 100644 --- a/src/core/capabilities.ts +++ b/src/core/capabilities.ts @@ -96,7 +96,7 @@ const COMMAND_CAPABILITY_MATRIX: Record = { device.kind === 'simulator', }, keyboard: { - // iOS only supports keyboard dismiss; status/get remains Android-only. + // iOS only supports keyboard dismiss/enter; status/get remains Android-only. apple: { simulator: true, device: true }, android: { emulator: true, device: true, unknown: true }, linux: LINUX_NONE, diff --git a/src/core/dispatch-context.ts b/src/core/dispatch-context.ts index 37387f42b..27b5b2f59 100644 --- a/src/core/dispatch-context.ts +++ b/src/core/dispatch-context.ts @@ -10,8 +10,17 @@ export type BatchStep = { runtime?: unknown; }; +export type MaestroRuntimeFlags = { + allowNonHittableCoordinateFallback?: boolean; + optional?: boolean; + runScriptEnv?: Record; +}; + export type CommandFlags = Omit & { batchSteps?: BatchStep[]; + clearAppState?: boolean; + launchArgs?: string[]; + maestro?: MaestroRuntimeFlags; replayBackend?: string; }; @@ -20,6 +29,8 @@ export type DispatchContext = ScreenshotDispatchFlags & { appBundleId?: string; activity?: string; launchConsole?: string; + launchArgs?: string[]; + clearAppState?: boolean; verbose?: boolean; logPath?: string; traceLogPath?: string; @@ -44,5 +55,6 @@ export type DispatchContext = ScreenshotDispatchFlags & { key: 'id' | 'label' | 'text' | 'value'; value: string; raw: string; + allowNonHittableCoordinateFallback?: boolean; }; }; diff --git a/src/core/dispatch-interactions.ts b/src/core/dispatch-interactions.ts index 530862e7f..76b78fa64 100644 --- a/src/core/dispatch-interactions.ts +++ b/src/core/dispatch-interactions.ts @@ -86,6 +86,15 @@ export async function handleFillCommand( positionals: string[], context: DispatchContext | undefined, ): Promise> { + if (context?.directElementSelector) { + return await handleDirectElementSelectorFill( + interactor, + context.directElementSelector, + positionals, + context, + ); + } + const x = Number(positionals[0]); const y = Number(positionals[1]); const text = positionals.slice(2).join(' '); @@ -97,6 +106,28 @@ export async function handleFillCommand( return { x, y, text, delayMs, ...successText(formatTextLengthMessage('Filled', text)) }; } +async function handleDirectElementSelectorFill( + interactor: Interactor, + selector: NonNullable, + positionals: string[], + context: DispatchContext, +): Promise> { + if (!interactor.fillElementSelector) { + throw new AppError('UNSUPPORTED_OPERATION', 'direct element selector fill is not supported'); + } + const text = positionals.join(' '); + if (!text) throw new AppError('INVALID_ARGS', 'fill requires text'); + const delayMs = requireIntInRange(context.delayMs ?? 0, 'delay-ms', 0, 10_000); + const result = await interactor.fillElementSelector(selector, text, delayMs); + return { + selector: selector.raw, + text, + delayMs, + ...(result ?? {}), + ...successText(formatTextLengthMessage('Filled', text)), + }; +} + export async function handlePressCommand( device: DeviceInfo, interactor: Interactor, diff --git a/src/core/dispatch.ts b/src/core/dispatch.ts index f28a3c6e8..05888e718 100644 --- a/src/core/dispatch.ts +++ b/src/core/dispatch.ts @@ -6,11 +6,12 @@ import { dismissAndroidKeyboard, getAndroidKeyboardState, } from '../platforms/android/device-input-state.ts'; +import { pressAndroidEnter } from '../platforms/android/input-actions.ts'; import { pushAndroidNotification } from '../platforms/android/notifications.ts'; import { getInteractor } from './interactors.ts'; import type { Interactor, RunnerContext } from './interactor-types.ts'; import { runIosRunnerCommand } from '../platforms/ios/runner-client.ts'; -import { pushIosNotification } from '../platforms/ios/apps.ts'; +import { clearIosSimulatorAppState, pushIosNotification } from '../platforms/ios/apps.ts'; import { isDeepLinkTarget } from './open-target.ts'; import { parseTriggerAppEventArgs, resolveAppEventUrl } from './app-events.ts'; import { @@ -43,7 +44,105 @@ import { parseDeviceRotation } from './device-rotation.ts'; export { resolveTargetDevice } from './dispatch-resolve.ts'; export type { BatchStep, CommandFlags, DispatchContext } from './dispatch-context.ts'; -// fallow-ignore-next-line complexity +type DispatchCommandHandlerParams = { + device: DeviceInfo; + interactor: Interactor; + positionals: string[]; + outPath?: string; + context?: DispatchContext; + runnerCtx: RunnerContext; +}; + +type DispatchCommandHandler = ( + params: DispatchCommandHandlerParams, +) => Promise | void> | Record | void; + +const DISPATCH_COMMAND_HANDLERS: Record = { + open: ({ device, interactor, positionals, context }) => + handleOpenCommand(device, interactor, positionals, context), + close: async ({ interactor, positionals }) => { + const app = positionals[0]; + if (!app) { + return { closed: 'session', ...successText('Closed session') }; + } + await interactor.close(app); + return { app, ...successText(`Closed: ${app}`) }; + }, + press: ({ device, interactor, positionals, context }) => + handlePressCommand(device, interactor, positionals, context), + swipe: ({ device, interactor, positionals, context }) => + handleSwipeCommand(device, interactor, positionals, context), + pan: ({ interactor, positionals }) => handlePanCommand(interactor, positionals), + fling: ({ interactor, positionals }) => handleFlingCommand(interactor, positionals), + longpress: ({ interactor, positionals }) => handleLongPressCommand(interactor, positionals), + focus: ({ interactor, positionals }) => handleFocusCommand(interactor, positionals), + type: ({ interactor, positionals, context }) => + handleTypeCommand(interactor, positionals, context), + fill: ({ interactor, positionals, context }) => + handleFillCommand(interactor, positionals, context), + scroll: ({ interactor, positionals, context }) => + handleScrollCommand(interactor, positionals, context), + pinch: ({ device, interactor, positionals, context }) => + handlePinchCommand(device, interactor, positionals, context), + 'rotate-gesture': ({ device, interactor, positionals }) => + handleRotateGestureCommand(device, interactor, positionals), + 'transform-gesture': ({ device, interactor, positionals }) => + handleTransformGestureCommand(device, interactor, positionals), + 'trigger-app-event': async ({ device, interactor, positionals, context }) => { + const { eventName, payload } = parseTriggerAppEventArgs(positionals); + const eventUrl = resolveAppEventUrl(device.platform, eventName, payload); + await interactor.open(eventUrl, { appBundleId: context?.appBundleId }); + return { + event: eventName, + eventUrl, + transport: 'deep-link', + ...successText(`Triggered app event: ${eventName}`), + }; + }, + screenshot: async ({ interactor, positionals, outPath, context }) => { + const positionalPath = positionals[0]; + const screenshotPath = positionalPath ?? outPath ?? `./screenshot-${Date.now()}.png`; + await fs.mkdir(pathModule.dirname(screenshotPath), { recursive: true }); + const screenshotOptions = screenshotOptionsFromFlags(context); + await interactor.screenshot(screenshotPath, { + appBundleId: context?.appBundleId, + fullscreen: screenshotOptions.fullscreen, + stabilize: screenshotOptions.stabilize, + surface: context?.surface, + }); + return { path: screenshotPath, ...successText(`Saved screenshot: ${screenshotPath}`) }; + }, + back: async ({ interactor, context }) => { + await interactor.back(context?.backMode); + return { action: 'back', mode: context?.backMode ?? 'in-app', ...successText('Back') }; + }, + home: async ({ interactor }) => { + await interactor.home(); + return { action: 'home', ...successText('Home') }; + }, + rotate: async ({ interactor, positionals }) => { + const orientation = parseDeviceRotation(positionals[0]); + await interactor.rotate(orientation); + return { + action: 'rotate', + orientation, + ...successText(`Rotated to ${orientation}`), + }; + }, + 'app-switcher': async ({ interactor }) => { + await interactor.appSwitcher(); + return { action: 'app-switcher', ...successText('Opened app switcher') }; + }, + clipboard: ({ interactor, positionals }) => handleClipboardCommand(interactor, positionals), + keyboard: ({ device, positionals, context, runnerCtx }) => + handleKeyboardCommand(device, positionals, context, runnerCtx), + settings: ({ device, interactor, positionals, context }) => + handleSettingsCommand(device, interactor, positionals, context), + push: ({ device, positionals, context }) => handlePushCommand(device, positionals, context), + snapshot: ({ interactor, context }) => handleSnapshotCommand(interactor, context), + read: ({ device, positionals, context }) => handleReadCommand(device, positionals, context), +}; + export async function dispatchCommand( device: DeviceInfo, command: string, @@ -71,98 +170,9 @@ export async function dispatchCommand( return await withDiagnosticTimer( 'platform_command', async () => { - switch (command) { - case 'open': - return handleOpenCommand(device, interactor, positionals, context); - case 'close': { - const app = positionals[0]; - if (!app) { - return { closed: 'session', ...successText('Closed session') }; - } - await interactor.close(app); - return { app, ...successText(`Closed: ${app}`) }; - } - case 'press': - return handlePressCommand(device, interactor, positionals, context); - case 'swipe': - return handleSwipeCommand(device, interactor, positionals, context); - case 'pan': - return handlePanCommand(interactor, positionals); - case 'fling': - return handleFlingCommand(interactor, positionals); - case 'longpress': - return handleLongPressCommand(interactor, positionals); - case 'focus': - return handleFocusCommand(interactor, positionals); - case 'type': - return handleTypeCommand(interactor, positionals, context); - case 'fill': - return handleFillCommand(interactor, positionals, context); - case 'scroll': - return handleScrollCommand(interactor, positionals, context); - case 'pinch': - return handlePinchCommand(device, interactor, positionals, context); - case 'rotate-gesture': - return handleRotateGestureCommand(device, interactor, positionals); - case 'transform-gesture': - return handleTransformGestureCommand(device, interactor, positionals); - case 'trigger-app-event': { - const { eventName, payload } = parseTriggerAppEventArgs(positionals); - const eventUrl = resolveAppEventUrl(device.platform, eventName, payload); - await interactor.open(eventUrl, { appBundleId: context?.appBundleId }); - return { - event: eventName, - eventUrl, - transport: 'deep-link', - ...successText(`Triggered app event: ${eventName}`), - }; - } - case 'screenshot': { - const positionalPath = positionals[0]; - const screenshotPath = positionalPath ?? outPath ?? `./screenshot-${Date.now()}.png`; - await fs.mkdir(pathModule.dirname(screenshotPath), { recursive: true }); - const screenshotOptions = screenshotOptionsFromFlags(context); - await interactor.screenshot(screenshotPath, { - appBundleId: context?.appBundleId, - fullscreen: screenshotOptions.fullscreen, - stabilize: screenshotOptions.stabilize, - surface: context?.surface, - }); - return { path: screenshotPath, ...successText(`Saved screenshot: ${screenshotPath}`) }; - } - case 'back': - await interactor.back(context?.backMode); - return { action: 'back', mode: context?.backMode ?? 'in-app', ...successText('Back') }; - case 'home': - await interactor.home(); - return { action: 'home', ...successText('Home') }; - case 'rotate': { - const orientation = parseDeviceRotation(positionals[0]); - await interactor.rotate(orientation); - return { - action: 'rotate', - orientation, - ...successText(`Rotated to ${orientation}`), - }; - } - case 'app-switcher': - await interactor.appSwitcher(); - return { action: 'app-switcher', ...successText('Opened app switcher') }; - case 'clipboard': - return handleClipboardCommand(interactor, positionals); - case 'keyboard': - return handleKeyboardCommand(device, positionals, context, runnerCtx); - case 'settings': - return handleSettingsCommand(device, interactor, positionals, context); - case 'push': - return handlePushCommand(device, positionals, context); - case 'snapshot': - return await handleSnapshotCommand(interactor, context); - case 'read': - return handleReadCommand(device, positionals, context); - default: - throw new AppError('INVALID_ARGS', `Unknown command: ${command}`); - } + const handler = DISPATCH_COMMAND_HANDLERS[command]; + if (!handler) throw new AppError('INVALID_ARGS', `Unknown command: ${command}`); + return await handler({ device, interactor, positionals, outPath, context, runnerCtx }); }, { command, @@ -217,6 +227,7 @@ async function handleOpenCommand( await interactor.open(app, { activity: context?.activity, appBundleId: context?.appBundleId, + launchArgs: context?.launchArgs, url, }); return { app, url, ...successText(`Opened: ${app}`) }; @@ -224,10 +235,32 @@ async function handleOpenCommand( if (launchConsole && isDeepLinkTarget(app)) { throw new AppError('INVALID_ARGS', LAUNCH_CONSOLE_DIRECT_APP_ONLY_MESSAGE); } + if (device.platform === 'android' && context?.launchArgs && context.launchArgs.length > 0) { + throw new AppError( + 'UNSUPPORTED_OPERATION', + 'Launch arguments are currently supported only on Apple platforms.', + ); + } + if (context?.clearAppState) { + if (isDeepLinkTarget(app)) { + throw new AppError( + 'INVALID_ARGS', + 'Clearing app state requires an app target, not a deep link.', + ); + } + if (device.platform !== 'ios' || device.kind !== 'simulator') { + throw new AppError( + 'UNSUPPORTED_OPERATION', + 'Clearing app state is currently supported only on iOS simulators.', + ); + } + await clearIosSimulatorAppState(device, app); + } await interactor.open(app, { activity: context?.activity, appBundleId: context?.appBundleId, launchConsole, + launchArgs: context?.launchArgs, }); return { app, ...(launchConsole ? { launchConsole } : {}), ...successText(`Opened: ${app}`) }; } @@ -266,65 +299,118 @@ async function handleKeyboardCommand( runnerCtx: RunnerContext, ): Promise> { const action = (positionals[0] ?? 'status').toLowerCase(); - if (action !== 'status' && action !== 'get' && action !== 'dismiss') { - throw new AppError('INVALID_ARGS', 'keyboard requires a subcommand: status, get, or dismiss'); + if (!isKeyboardAction(action)) { + throw new AppError( + 'INVALID_ARGS', + 'keyboard requires a subcommand: status, get, dismiss, enter, or return', + ); } if (positionals.length > 1) { throw new AppError('INVALID_ARGS', 'keyboard accepts at most one subcommand argument'); } if (device.platform === 'android') { - if (action === 'dismiss') { - const result = await dismissAndroidKeyboard(device); - return { - platform: 'android', - action: 'dismiss', - attempts: result.attempts, - wasVisible: result.wasVisible, - dismissed: result.dismissed, - visible: result.visible, - inputType: result.inputType, - type: result.type, - inputMethodPackage: result.inputMethodPackage, - focusedPackage: result.focusedPackage, - focusedResourceId: result.focusedResourceId, - inputOwner: result.inputOwner, - }; - } - const state = await getAndroidKeyboardState(device); + return await handleAndroidKeyboardCommand(device, action); + } + if (device.platform === 'ios') { + return await handleIosKeyboardCommand(device, action, context, runnerCtx); + } + throw new AppError('UNSUPPORTED_OPERATION', 'keyboard is supported only on Android and iOS'); +} + +function isKeyboardAction( + action: string, +): action is 'status' | 'get' | 'dismiss' | 'enter' | 'return' { + return ( + action === 'status' || + action === 'get' || + action === 'dismiss' || + action === 'enter' || + action === 'return' + ); +} + +async function handleAndroidKeyboardCommand( + device: DeviceInfo, + action: 'status' | 'get' | 'dismiss' | 'enter' | 'return', +): Promise> { + if (action === 'enter' || action === 'return') { + await pressAndroidEnter(device); return { platform: 'android', - action: 'status', - visible: state.visible, - inputType: state.inputType, - type: state.type, - inputMethodPackage: state.inputMethodPackage, - focusedPackage: state.focusedPackage, - focusedResourceId: state.focusedResourceId, - inputOwner: state.inputOwner, + action: 'enter', + ...successText('Keyboard enter pressed'), }; } - if (device.platform === 'ios') { - if (action !== 'dismiss') { - throw new AppError( - 'UNSUPPORTED_OPERATION', - 'keyboard status/get is currently supported only on Android; use keyboard dismiss on iOS', - ); - } + if (action === 'dismiss') { + const result = await dismissAndroidKeyboard(device); + return { + platform: 'android', + action: 'dismiss', + attempts: result.attempts, + wasVisible: result.wasVisible, + dismissed: result.dismissed, + visible: result.visible, + inputType: result.inputType, + type: result.type, + inputMethodPackage: result.inputMethodPackage, + focusedPackage: result.focusedPackage, + focusedResourceId: result.focusedResourceId, + inputOwner: result.inputOwner, + }; + } + const state = await getAndroidKeyboardState(device); + return { + platform: 'android', + action: 'status', + visible: state.visible, + inputType: state.inputType, + type: state.type, + inputMethodPackage: state.inputMethodPackage, + focusedPackage: state.focusedPackage, + focusedResourceId: state.focusedResourceId, + inputOwner: state.inputOwner, + }; +} + +async function handleIosKeyboardCommand( + device: DeviceInfo, + action: 'status' | 'get' | 'dismiss' | 'enter' | 'return', + context: DispatchContext | undefined, + runnerCtx: RunnerContext, +): Promise> { + if (action !== 'dismiss' && action !== 'enter' && action !== 'return') { + throw new AppError( + 'UNSUPPORTED_OPERATION', + 'keyboard status/get is currently supported only on Android; use keyboard dismiss or enter on iOS', + ); + } + if (action === 'enter' || action === 'return') { const result = await runIosRunnerCommand( device, - { command: 'keyboardDismiss', appBundleId: context?.appBundleId }, + { command: 'keyboardReturn', appBundleId: context?.appBundleId }, runnerCtx, ); return { platform: 'ios', - action: 'dismiss', - wasVisible: result.wasVisible, - dismissed: result.dismissed, + action: 'enter', visible: result.visible, - ...successText(result.dismissed ? 'Keyboard dismissed' : 'Keyboard already hidden'), + wasVisible: result.wasVisible, + ...successText('Keyboard enter pressed'), }; } - throw new AppError('UNSUPPORTED_OPERATION', 'keyboard is supported only on Android and iOS'); + const result = await runIosRunnerCommand( + device, + { command: 'keyboardDismiss', appBundleId: context?.appBundleId }, + runnerCtx, + ); + return { + platform: 'ios', + action: 'dismiss', + wasVisible: result.wasVisible, + dismissed: result.dismissed, + visible: result.visible, + ...successText(result.dismissed ? 'Keyboard dismissed' : 'Keyboard already hidden'), + }; } async function handleSettingsCommand( diff --git a/src/core/interactor-types.ts b/src/core/interactor-types.ts index 01653a5e9..2ddf377a8 100644 --- a/src/core/interactor-types.ts +++ b/src/core/interactor-types.ts @@ -29,6 +29,7 @@ export type ScreenshotOptions = { export type ElementSelectorTapOptions = { key: 'id' | 'label' | 'text' | 'value'; value: string; + allowNonHittableCoordinateFallback?: boolean; }; export type SnapshotOptions = BaseSnapshotOptions & { @@ -44,7 +45,13 @@ export type SnapshotResult = Omit & export type Interactor = { open( app: string, - options?: { activity?: string; appBundleId?: string; launchConsole?: string; url?: string }, + options?: { + activity?: string; + appBundleId?: string; + launchConsole?: string; + launchArgs?: string[]; + url?: string; + }, ): Promise; openDevice(): Promise; close(app: string): Promise; @@ -75,6 +82,11 @@ export type Interactor = { longPress(x: number, y: number, durationMs?: number): Promise | void>; focus(x: number, y: number): Promise | void>; type(text: string, delayMs?: number): Promise; + fillElementSelector?( + selector: ElementSelectorTapOptions, + text: string, + delayMs?: number, + ): Promise | void>; fill( x: number, y: number, diff --git a/src/core/interactors/apple.ts b/src/core/interactors/apple.ts index a841b8c99..744adcb01 100644 --- a/src/core/interactors/apple.ts +++ b/src/core/interactors/apple.ts @@ -30,6 +30,7 @@ export function createAppleInteractor( openIosApp(device, app, { appBundleId: options?.appBundleId, launchConsole: options?.launchConsole, + launchArgs: options?.launchArgs, url: options?.url, }), openDevice: () => openIosDevice(device), diff --git a/src/daemon-client.ts b/src/daemon-client.ts index ba17f166e..2faf868c8 100644 --- a/src/daemon-client.ts +++ b/src/daemon-client.ts @@ -117,7 +117,7 @@ export async function sendToDaemon(req: Omit): Promise await ensureDaemon(settings), @@ -168,6 +168,14 @@ export async function sendToDaemon(req: Omit): Promise): number | undefined { + if (req.command === 'test') return undefined; + if (req.command === 'replay' && typeof req.flags?.timeoutMs === 'number') { + return req.flags.timeoutMs; + } + return REQUEST_TIMEOUT_MS; +} + export async function openApp(options: OpenAppOptions = {}): Promise { const { session = 'default', diff --git a/src/daemon/__tests__/context.test.ts b/src/daemon/__tests__/context.test.ts index 090314199..92c1b6339 100644 --- a/src/daemon/__tests__/context.test.ts +++ b/src/daemon/__tests__/context.test.ts @@ -14,6 +14,12 @@ test('contextFromFlags forwards scroll pixels from CLI flags', () => { assert.equal(context.pixels, 240); }); +test('contextFromFlags forwards generic app-state clearing', () => { + const flags: CommandFlags = { clearAppState: true }; + const context = contextFromFlags('/tmp/agent-device.log', flags); + assert.equal(context.clearAppState, true); +}); + test('contextFromFlags forwards screenshot flags from CLI flags', () => { const flags: CommandFlags = { screenshotFullscreen: true, diff --git a/src/daemon/context.ts b/src/daemon/context.ts index 05c0864d4..7947241d0 100644 --- a/src/daemon/context.ts +++ b/src/daemon/context.ts @@ -8,6 +8,9 @@ import { getDiagnosticsMeta } from '../utils/diagnostics.ts'; export type DaemonCommandContext = DispatchContext & ScreenshotRuntimeFlags; +// Flat compatibility mapper: keeping each CLI flag visible here makes request +// context drift easier to spot than splitting the same optional fields apart. +// fallow-ignore-next-line complexity export function contextFromFlags( logPath: string, flags: CommandFlags | undefined, @@ -21,6 +24,8 @@ export function contextFromFlags( appBundleId, activity: flags?.activity, launchConsole: flags?.launchConsole, + launchArgs: flags?.launchArgs, + clearAppState: flags?.clearAppState, verbose: flags?.verbose, logPath, traceLogPath, diff --git a/src/daemon/direct-ios-selector.ts b/src/daemon/direct-ios-selector.ts index 5063f6632..60dfe075f 100644 --- a/src/daemon/direct-ios-selector.ts +++ b/src/daemon/direct-ios-selector.ts @@ -6,6 +6,7 @@ export type DirectIosSelectorTarget = { key: 'id' | 'label' | 'text' | 'value'; value: string; raw: string; + allowNonHittableCoordinateFallback?: boolean; }; export function readSimpleIosSelectorTarget(params: { diff --git a/src/daemon/handlers/__tests__/find.test.ts b/src/daemon/handlers/__tests__/find.test.ts index 7efbca153..8c9b92c0d 100644 --- a/src/daemon/handlers/__tests__/find.test.ts +++ b/src/daemon/handlers/__tests__/find.test.ts @@ -95,6 +95,7 @@ test('handleFindCommands click returns deterministic metadata across locator var expectedLocator: 'any', expectedQuery: 'Increment', expectedCoordinates: { x: 100, y: 50 }, + expectedRef: '@e2', }, ]; @@ -104,7 +105,7 @@ test('handleFindCommands click returns deterministic metadata across locator var if (!response.ok) return; const data = response.data as Record; expect(Object.keys(data).sort()).toEqual(scenario.expectedKeys); - expect(data.ref).toBe('@e1'); + expect(data.ref).toBe(scenario.expectedRef); expect(data.locator).toBe(scenario.expectedLocator); expect(data.query).toBe(scenario.expectedQuery); @@ -117,10 +118,101 @@ test('handleFindCommands click returns deterministic metadata across locator var } expect(invokeCalls.length).toBe(1); - expect(invokeCalls[0].positionals?.[0]).toBe('@e1'); + expect(invokeCalls[0].positionals?.[0]).toBe(scenario.expectedRef); } }); +test('handleFindCommands click prefers on-screen duplicate text matches', async () => { + const { response, invokeCalls } = await runFindClickScenario({ + positionals: ['Sign in', 'click'], + nodes: [ + { + index: 0, + ref: 'e1', + type: 'Application', + hittable: true, + rect: { x: 0, y: 0, width: 440, height: 956 }, + }, + { + index: 1, + ref: 'e2', + type: 'Button', + label: 'Sign in', + hittable: false, + rect: { x: -199, y: 186, width: 70, height: 33 }, + parentIndex: 0, + }, + { + index: 2, + ref: 'e3', + type: 'Button', + label: 'Sign in', + hittable: false, + rect: { x: 40, y: 870, width: 360, height: 44 }, + parentIndex: 0, + }, + ], + }); + + expect(response.ok).toBe(true); + expect(invokeCalls[0].positionals?.[0]).toBe('@e3'); +}); + +test('handleFindCommands click prefers semantic controls over matching containers', async () => { + const { response, invokeCalls } = await runFindClickScenario({ + positionals: ['Later', 'click'], + flags: { findFirst: true }, + nodes: [ + { + index: 0, + ref: 'e1', + type: 'Application', + hittable: true, + rect: { x: 0, y: 0, width: 440, height: 956 }, + }, + { + index: 1, + ref: 'e2', + type: 'Element(5)', + label: 'Dialog', + hittable: true, + rect: { x: 60, y: 356, width: 320, height: 272 }, + parentIndex: 0, + }, + { + index: 2, + ref: 'e3', + type: 'ScrollView', + label: 'Later', + hittable: false, + rect: { x: 60, y: 548, width: 320, height: 80 }, + parentIndex: 1, + }, + { + index: 3, + ref: 'e4', + type: 'Other', + label: 'Later', + hittable: false, + rect: { x: 76, y: 564, width: 288, height: 48 }, + parentIndex: 2, + }, + { + index: 4, + ref: 'e5', + type: 'Button', + label: 'Later', + hittable: false, + rect: { x: 76, y: 564, width: 140, height: 48 }, + parentIndex: 3, + }, + ], + }); + + expect(response.ok).toBe(true); + expect(invokeCalls[0].positionals?.[0]).toBe('@e5'); +}); + test('handleFindCommands wait bypasses snapshot cache while Android freshness recovery is active', async () => { const sessionName = 'android-find-wait'; const session: SessionState = { diff --git a/src/daemon/handlers/__tests__/interaction-touch-targets.test.ts b/src/daemon/handlers/__tests__/interaction-touch-targets.test.ts index eed6d7af5..fc29098f4 100644 --- a/src/daemon/handlers/__tests__/interaction-touch-targets.test.ts +++ b/src/daemon/handlers/__tests__/interaction-touch-targets.test.ts @@ -52,6 +52,19 @@ test('parseFillTarget reads selector text through shared fill codec', () => { }); }); +test('parseFillTarget preserves selector text whitespace', () => { + const parsed = parseFillTarget(['label="Command"', 'submit\n']); + + expect(parsed).toEqual({ + ok: true, + target: { + kind: 'selector', + selector: 'label="Command"', + }, + text: 'submit\n', + }); +}); + test('parseFillTarget rejects invalid coordinates instead of treating them as a point', () => { const parsed = parseFillTarget(['10', 'not-y', 'text']); diff --git a/src/daemon/handlers/__tests__/interaction.test.ts b/src/daemon/handlers/__tests__/interaction.test.ts index 60e4eb7d9..0fdbf8dea 100644 --- a/src/daemon/handlers/__tests__/interaction.test.ts +++ b/src/daemon/handlers/__tests__/interaction.test.ts @@ -410,6 +410,86 @@ test('click simple iOS id selector uses direct runner selector tap without snaps } }); +test('fill simple iOS id selector uses direct runner selector fill without snapshot coordinates', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'ios-direct-selector-fill'; + sessionStore.set(sessionName, makeIosSession(sessionName, { appBundleId: 'com.example.app' })); + + mockDispatch.mockResolvedValue({ + message: 'filled', + x: 439.5, + y: 100.5, + referenceWidth: 440, + referenceHeight: 956, + }); + + const response = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'fill', + positionals: ['id="email"', 'ada@example.com'], + flags: { delayMs: 25 }, + }, + sessionName, + sessionStore, + contextFromFlags, + }); + + expect(response?.ok).toBe(true); + expect(mockDispatch).toHaveBeenCalledTimes(1); + expect(mockDispatch.mock.calls[0]?.[1]).toBe('fill'); + expect(mockDispatch.mock.calls[0]?.[2]).toEqual(['ada@example.com']); + const context = mockDispatch.mock.calls[0]?.[4] as Record; + expect(context.directElementSelector).toEqual({ + key: 'id', + value: 'email', + raw: 'id="email"', + }); + expect(context.delayMs).toBe(25); + if (response?.ok) { + expect(response.data?.selector).toBe('id="email"'); + expect(response.data?.text).toBe('ada@example.com'); + } +}); + +test('click simple iOS selector forwards Maestro non-hittable coordinate fallback', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'ios-maestro-selector-fallback'; + sessionStore.set(sessionName, makeIosSession(sessionName, { appBundleId: 'com.example.app' })); + + mockDispatch.mockResolvedValue({ + message: 'tapped via non-hittable coordinate fallback', + x: 439.5, + y: 101.5, + referenceWidth: 440, + referenceHeight: 956, + }); + + const response = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'click', + positionals: ['id="e2eSignInAlice"'], + flags: { maestro: { allowNonHittableCoordinateFallback: true } }, + }, + sessionName, + sessionStore, + contextFromFlags, + }); + + expect(response?.ok).toBe(true); + const pressCalls = mockDispatch.mock.calls.filter((call) => call[1] === 'press'); + expect(pressCalls.length).toBe(1); + expect((pressCalls[0]?.[4] as Record)?.directElementSelector).toEqual({ + key: 'id', + value: 'e2eSignInAlice', + raw: 'id="e2eSignInAlice"', + allowNonHittableCoordinateFallback: true, + }); +}); + test('click simple iOS id selector falls back to snapshot coordinates when direct tap fails', async () => { const sessionStore = makeSessionStore(); const sessionName = 'ios-direct-selector-fallback'; diff --git a/src/daemon/handlers/__tests__/session-replay-vars.test.ts b/src/daemon/handlers/__tests__/session-replay-vars.test.ts index db01e2b9c..20b71eb70 100644 --- a/src/daemon/handlers/__tests__/session-replay-vars.test.ts +++ b/src/daemon/handlers/__tests__/session-replay-vars.test.ts @@ -32,6 +32,7 @@ type CapturedInvocation = { async function runReplayFixture(params: { label: string; script: string; + files?: Record; flags?: CommandFlags; invoke?: (req: DaemonRequest) => Promise; }): Promise<{ @@ -41,11 +42,15 @@ async function runReplayFixture(params: { scriptPath: string; }> { const root = fs.mkdtempSync(path.join(os.tmpdir(), `agent-device-replay-${params.label}-`)); + for (const [name, contents] of Object.entries(params.files ?? {})) { + fs.writeFileSync(path.join(root, name), contents); + } const scriptPath = path.join(root, 'flow.ad'); fs.writeFileSync(scriptPath, params.script); const calls: CapturedInvocation[] = []; - const defaultInvoke = async (req: DaemonRequest): Promise => { + const invoke = async (req: DaemonRequest): Promise => { calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (params.invoke) return await params.invoke(req); return { ok: true, data: {} }; }; const response = await runReplayScriptFile({ @@ -60,7 +65,7 @@ async function runReplayFixture(params: { sessionName: 's', logPath: path.join(root, 'log'), sessionStore: new SessionStore(path.join(root, 'state')), - invoke: params.invoke ?? defaultInvoke, + invoke, }); return { response, calls, root, scriptPath }; } @@ -434,11 +439,1027 @@ test('runReplayScriptFile applies CLI env overrides before Maestro compat mappin replayShellEnv: { AD_VAR_BUTTON_ID: 'shell-button' }, replayEnv: ['APP_ID=cli-app'], }, + invoke: async (req) => { + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + identifier: 'shell-button', + rect: { x: 20, y: 40, width: 120, height: 44 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, }); assert.equal(response.ok, true); assert.deepEqual(calls[0]?.positionals, ['cli-app']); - assert.deepEqual(calls[1]?.positionals, ['id="shell-button"']); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['open', ['cli-app']], + ['snapshot', []], + ['click', ['80', '62']], + ], + ); +}); + +test('runReplayScriptFile runs Maestro runScript in replay order and exposes output variables', async () => { + const { response, calls } = await runReplayFixture({ + label: 'maestro-runscript-runtime', + files: { + 'setup.js': ` +var res = {body: '{"appviewDid":"did:plc:test"}'} +output.result = SERVER_PATH + ':' + json(res.body).appviewDid +`, + }, + script: [ + 'appId: demo.app', + '---', + '- runScript:', + ' file: ./setup.js', + ' env:', + ' SERVER_PATH: local', + '- inputText: ${output.result}', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [['type', ['local:did:plc:test']]], + ); +}); + +test('runReplayScriptFile reports Maestro runScript failures at the runScript step', async () => { + const { response, calls } = await runReplayFixture({ + label: 'maestro-runscript-fail', + files: { + 'setup.js': `output.result = http.post('http://127.0.0.1:1').body`, + }, + script: ['appId: demo.app', '---', '- runScript: ./setup.js', '- inputText: never', ''].join( + '\n', + ), + flags: { replayBackend: 'maestro' }, + }); + + assert.equal(response.ok, false); + if (!response.ok) { + assert.match(response.error.message, /Replay failed at step 1/); + assert.match(response.error.message, /runScript failed/); + assert.match(response.error.message, /http\.post failed/); + } + assert.equal(calls.length, 0); +}); + +test('runReplayScriptFile rejects Maestro runScript output keys containing dots', async () => { + const { response, calls } = await runReplayFixture({ + label: 'maestro-runscript-dotted-output', + files: { + 'setup.js': `output['nested.value'] = 'ambiguous'`, + }, + script: ['appId: demo.app', '---', '- runScript: ./setup.js', '- inputText: never', ''].join( + '\n', + ), + flags: { replayBackend: 'maestro' }, + }); + + assert.equal(response.ok, false); + if (!response.ok) { + assert.match(response.error.message, /Replay failed at step 1/); + assert.match(response.error.message, /output key cannot contain/); + } + assert.equal(calls.length, 0); +}); + +test('runReplayScriptFile retries Maestro scrollUntilVisible with scroll probes', async () => { + const calls: CapturedInvocation[] = []; + let waitAttempts = 0; + const { response } = await runReplayFixture({ + label: 'maestro-scroll-until-visible', + script: [ + 'appId: demo.app', + '---', + '- scrollUntilVisible:', + ' element: Discover', + ' direction: UP', + ' timeout: 1200', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'scroll') return { ok: true, data: {} }; + if (req.command === 'find') { + return { + ok: false, + error: { code: 'COMMAND_FAILED', message: 'find wait timed out' }, + }; + } + waitAttempts += 1; + if (waitAttempts === 3) return { ok: true, data: { waitedMs: 1100 } }; + return { + ok: false, + error: { code: 'COMMAND_FAILED', message: 'wait timed out' }, + }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['wait', ['label="Discover" || text="Discover" || id="Discover"', '500']], + ['find', ['Discover', 'wait', '500']], + ['scroll', ['up']], + ['wait', ['label="Discover" || text="Discover" || id="Discover"', '500']], + ['find', ['Discover', 'wait', '500']], + ['scroll', ['up']], + ['wait', ['label="Discover" || text="Discover" || id="Discover"', '200']], + ], + ); +}); + +test('runReplayScriptFile lets Maestro scrollUntilVisible use fuzzy visible text matching', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-scroll-until-visible-fuzzy-text', + script: ['appId: demo.app', '---', '- scrollUntilVisible:', ' element: Discover', ''].join( + '\n', + ), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'find') return { ok: true, data: { found: true } }; + return { + ok: false, + error: { code: 'COMMAND_FAILED', message: 'wait timed out' }, + }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['wait', ['label="Discover" || text="Discover" || id="Discover"', '500']], + ['find', ['Discover', 'wait', '500']], + ], + ); +}); + +test('runReplayScriptFile lets Maestro tapOn use fuzzy visible text matching', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-visible-text-fuzzy', + script: ['appId: demo.app', '---', '- tapOn: Discover', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'find') return { ok: true, data: { found: true } }; + return { + ok: false, + error: { code: 'COMMAND_FAILED', message: 'Selector did not match' }, + }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['find', ['Discover', 'click']], + ], + ); + assert.equal(calls[0]?.flags?.noRecord, true); + assert.equal(calls[1]?.flags?.findFirst, true); +}); + +test('runReplayScriptFile lets exact Maestro text tapOn win before fuzzy matching', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-visible-text-exact-before-fuzzy', + script: ['appId: demo.app', '---', '- tapOn: Mute accounts', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + label: 'Block accounts', + rect: { x: 10, y: 600, width: 240, height: 44 }, + }, + { + index: 2, + label: 'Mute accounts', + rect: { x: 10, y: 540, width: 240, height: 44 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['click', ['130', '562']], + ], + ); +}); + +test('runReplayScriptFile prefers on-screen Maestro text tapOn matches', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-visible-text-onscreen', + script: ['appId: demo.app', '---', '- tapOn: Sign in', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + type: 'Button', + label: 'Sign in', + rect: { x: -328, y: 182, width: 328, height: 42 }, + }, + { + index: 2, + type: 'Button', + label: 'Sign in', + rect: { x: 56, y: 842, width: 328, height: 56 }, + }, + ], + metadata: { referenceWidth: 440, referenceHeight: 956 }, + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['click', ['220', '870']], + ], + ); +}); + +test('runReplayScriptFile taps Maestro text near the label in large action containers', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-visible-text-action-container', + script: ['appId: demo.app', '---', '- tapOn: Mute accounts', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + type: 'ScrollView', + label: 'Mute accounts', + rect: { x: 8, y: 805, width: 424, height: 93 }, + }, + { + index: 2, + parentIndex: 1, + type: 'Other', + label: 'Block accounts', + rect: { x: 31, y: 835, width: 377, height: 42 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['click', ['92', '829']], + ], + ); +}); + +test('runReplayScriptFile prefers actionable Maestro tapOn matches over broad ancestors', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-prefers-actionable-match', + script: ['appId: demo.app', '---', '- tapOn: New list', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + type: 'Other', + label: 'New list', + rect: { x: 0, y: 0, width: 440, height: 956 }, + }, + { + index: 2, + type: 'Button', + label: 'New list', + rect: { x: 349, y: 67, width: 75, height: 33 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['click', ['387', '84']], + ], + ); +}); + +test('runReplayScriptFile treats absent Maestro assertNotVisible targets as passing', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-assert-not-visible-absent', + script: ['appId: demo.app', '---', '- assertNotVisible: Feeds ✨', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + return { + ok: false, + error: { code: 'ELEMENT_NOT_FOUND', message: 'Selector did not match' }, + }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [['is', ['visible', 'label="Feeds ✨" || text="Feeds ✨" || id="Feeds ✨"']]], + ); + assert.equal(calls[0]?.flags?.noRecord, true); +}); + +test('runReplayScriptFile retries Maestro fuzzy tapOn without raw selector fallback', async () => { + const calls: CapturedInvocation[] = []; + let findAttempts = 0; + const { response } = await runReplayFixture({ + label: 'maestro-tap-visible-text-fuzzy-retry', + script: ['appId: demo.app', '---', '- tapOn: Discover', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'find') { + findAttempts += 1; + if (findAttempts === 2) return { ok: true, data: { found: true } }; + } + return { + ok: false, + error: { code: 'ELEMENT_NOT_FOUND', message: 'element not found' }, + }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['find', ['Discover', 'click']], + ['snapshot', []], + ['find', ['Discover', 'click']], + ], + ); +}); + +test('runReplayScriptFile lets optional Maestro fuzzy tapOn click first visible match', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-visible-text-optional-first-match', + script: [ + 'appId: demo.app', + '---', + '- tapOn:', + ' text: Later', + ' optional: true', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'find' && req.flags?.findFirst === true) { + return { ok: true, data: { ref: '@e4', x: 220, y: 720 } }; + } + return { + ok: false, + error: { code: 'AMBIGUOUS_MATCH', message: 'matched multiple elements' }, + }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['find', ['Later', 'click']], + ], + ); + assert.equal(calls[1]?.flags?.findFirst, true); +}); + +test('runReplayScriptFile resolves Maestro percentage point taps from snapshot size', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-point-percent', + script: ['appId: demo.app', '---', '- tapOn:', ' point: 20%,20%', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 0, + type: 'application', + rect: { x: 0, y: 0, width: 1000, height: 2000 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['click', ['200', '400']], + ], + ); + assert.equal(calls[0]?.flags?.noRecord, true); +}); + +test('runReplayScriptFile retries Maestro id tapOn through snapshot coordinates', async () => { + const calls: CapturedInvocation[] = []; + let snapshotAttempts = 0; + const { response } = await runReplayFixture({ + label: 'maestro-tap-on-retry', + script: ['appId: demo.app', '---', '- tapOn:', ' id: delayedButton', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + snapshotAttempts += 1; + if (snapshotAttempts === 3) { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + identifier: 'delayedButton', + rect: { x: 20, y: 40, width: 120, height: 44 }, + }, + ], + }, + }; + } + return { + ok: false, + error: { code: 'ELEMENT_NOT_FOUND', message: 'element not found' }, + }; + } + if (req.command === 'click') return { ok: true, data: {} }; + return { + ok: false, + error: { code: 'ELEMENT_NOT_FOUND', message: 'element not found' }, + }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['snapshot', []], + ['snapshot', []], + ['click', ['80', '62']], + ], + ); +}); + +test('runReplayScriptFile resolves Maestro tapOn index and childOf from snapshots', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-index-childof', + script: [ + 'appId: demo.app', + '---', + '- tapOn:', + ' id: likeBtn', + ' childOf:', + ' id: postThreadItem-by-bob.test', + '- tapOn:', + ' id: postDropdownBtn', + ' index: 1', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { index: 1, identifier: 'postThreadItem-by-alice.test' }, + { + index: 2, + parentIndex: 1, + identifier: 'likeBtn', + rect: { x: 10, y: 10, width: 40, height: 20 }, + }, + { index: 10, identifier: 'postThreadItem-by-bob.test' }, + { + index: 11, + parentIndex: 10, + identifier: 'likeBtn', + rect: { x: 20, y: 120, width: 40, height: 20 }, + }, + { + index: 20, + identifier: 'postDropdownBtn', + rect: { x: 100, y: 200, width: 40, height: 20 }, + }, + { + index: 21, + identifier: 'postDropdownBtn', + rect: { x: 200, y: 300, width: 40, height: 20 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['click', ['40', '130']], + ['snapshot', []], + ['click', ['220', '310']], + ], + ); + assert.equal(calls[0]?.flags?.noRecord, true); +}); + +test('runReplayScriptFile lets snapshot id tap handle Maestro one-point edge controls', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-edge-rect', + script: ['appId: demo.app', '---', '- tapOn:', ' id: e2eSignInAlice', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + identifier: 'e2eSignInAlice', + rect: { x: 0, y: 0, width: 1, height: 1 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['click', ['0', '0']], + ], + ); +}); + +test('runReplayScriptFile keeps Maestro text-entry tapOn on the snapshot path', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-input-text-snapshot', + script: [ + 'appId: demo.app', + '---', + '- tapOn:', + ' id: editListNameInput', + '- inputText: Muted Users', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + identifier: 'editListNameInput', + rect: { x: 20, y: 100, width: 200, height: 40 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['click', ['120', '120']], + ['type', ['Muted Users']], + ], + ); + assert.equal(calls[0]?.flags?.noRecord, true); +}); + +test('runReplayScriptFile resolves Maestro swipe.label from a labeled element rect', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-swipe-label', + script: [ + 'appId: demo.app', + '---', + '- swipe:', + ' label: Thread body', + ' direction: UP', + ' duration: 400', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + label: 'Thread body', + rect: { x: 10, y: 100, width: 200, height: 300 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['swipe', ['110', '250', '110', '8', '400']], + ], + ); +}); + +test('runReplayScriptFile maps Maestro enter to keyboard enter', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-press-enter', + script: ['appId: demo.app', '---', '- pressKey: Enter', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [['keyboard', ['enter']]], + ); +}); + +test('runReplayScriptFile waits for Maestro animation snapshots to stabilize', async () => { + const calls: CapturedInvocation[] = []; + let snapshots = 0; + const { response } = await runReplayFixture({ + label: 'maestro-wait-animation-stable', + script: ['appId: demo.app', '---', '- waitForAnimationToEnd', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + snapshots += 1; + const y = snapshots === 1 ? 100 : 120; + return { + ok: true, + data: { + nodes: [ + { + index: 1, + label: 'Animating', + rect: { x: 10, y, width: 100, height: 40 }, + }, + ], + }, + }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['snapshot', []], + ['snapshot', []], + ], + ); + assert.equal(calls[0]?.flags?.noRecord, true); +}); + +test('runReplayScriptFile falls back to newline type when keyboard enter is unsupported', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-press-enter-fallback', + script: ['appId: demo.app', '---', '- pressKey: Enter', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'keyboard') { + return { ok: false, error: { code: 'UNSUPPORTED_OPERATION', message: 'unsupported' } }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['keyboard', ['enter']], + ['type', ['\n']], + ], + ); +}); + +test('runReplayScriptFile skips Maestro runFlow.when.visible commands when absent', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-run-flow-when-visible-skip', + script: [ + 'appId: demo.app', + '---', + '- runFlow:', + ' when:', + ' visible: Continue', + ' commands:', + ' - tapOn: Continue', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + return { + ok: false, + error: { + code: 'COMMAND_FAILED', + message: 'not visible', + details: { command: 'is', reason: 'selector_not_found' }, + }, + }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [['is', ['visible', 'label="Continue" || text="Continue" || id="Continue"']]], + ); +}); + +test('runReplayScriptFile skips Maestro runFlow.when.visible commands on false predicate', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-run-flow-when-visible-false-skip', + script: [ + 'appId: demo.app', + '---', + '- runFlow:', + ' when:', + ' visible: Continue', + ' commands:', + ' - tapOn: Continue', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + return { ok: true, data: { pass: false } }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [['is', ['visible', 'label="Continue" || text="Continue" || id="Continue"']]], + ); +}); + +test('runReplayScriptFile propagates Maestro runFlow.when runtime errors', async () => { + const { response } = await runReplayFixture({ + label: 'maestro-run-flow-when-visible-runtime-error', + script: [ + 'appId: demo.app', + '---', + '- runFlow:', + ' when:', + ' visible: Continue', + ' commands:', + ' - tapOn: Continue', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async () => ({ + ok: false, + error: { code: 'UNKNOWN', message: 'fetch failed' }, + }), + }); + + assert.equal(response.ok, false); + if (!response.ok) { + assert.equal(response.error.code, 'UNKNOWN'); + assert.match(response.error.message, /fetch failed/); + } +}); + +test('runReplayScriptFile propagates Maestro runFlow.when COMMAND_FAILED errors without condition-miss details', async () => { + const { response } = await runReplayFixture({ + label: 'maestro-run-flow-when-visible-command-error', + script: [ + 'appId: demo.app', + '---', + '- runFlow:', + ' when:', + ' visible: Continue', + ' commands:', + ' - tapOn: Continue', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async () => ({ + ok: false, + error: { code: 'COMMAND_FAILED', message: 'snapshot failed' }, + }), + }); + + assert.equal(response.ok, false); + if (!response.ok) { + assert.equal(response.error.code, 'COMMAND_FAILED'); + assert.match(response.error.message, /snapshot failed/); + } +}); + +test('runReplayScriptFile runs Maestro runFlow.when.visible commands when present', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-run-flow-when-visible-run', + script: [ + 'appId: demo.app', + '---', + '- runFlow:', + ' when:', + ' visible: Continue', + ' commands:', + ' - tapOn: Continue', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'is') return { ok: true, data: { pass: true } }; + if (req.command === 'click') { + return { + ok: false, + error: { code: 'COMMAND_FAILED', message: 'Selector did not match' }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['is', ['visible', 'label="Continue" || text="Continue" || id="Continue"']], + ['snapshot', []], + ['find', ['Continue', 'click']], + ], + ); +}); + +test('runReplayScriptFile runs nested Maestro runtime commands inside runFlow.when', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-run-flow-when-nested-runtime', + script: [ + 'appId: demo.app', + '---', + '- runFlow:', + ' when:', + ' visible: Feed', + ' commands:', + ' - scrollUntilVisible:', + ' element: Done', + ' direction: DOWN', + ' timeout: 500', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'is') return { ok: true, data: { pass: true } }; + if (req.command === 'wait') return { ok: true, data: { found: true } }; + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['is', ['visible', 'label="Feed" || text="Feed" || id="Feed"']], + ['wait', ['label="Done" || text="Done" || id="Done"', '500']], + ], + ); }); test('runReplayScriptFile reads shell env from request (client-collected), not daemon process.env', async () => { diff --git a/src/daemon/handlers/__tests__/session-replay.test.ts b/src/daemon/handlers/__tests__/session-replay.test.ts index ae98d0939..74e656d03 100644 --- a/src/daemon/handlers/__tests__/session-replay.test.ts +++ b/src/daemon/handlers/__tests__/session-replay.test.ts @@ -1,6 +1,7 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; -import { buildReplayActionFlags, withReplayFailureContext } from '../session-replay-runtime.ts'; +import { buildReplayActionFlags } from '../session-replay-action-runtime.ts'; +import { withReplayFailureContext } from '../session-replay-runtime.ts'; import { buildNestedReplayFlags } from '../session-replay.ts'; test('buildReplayActionFlags keeps allowed parent flags only', () => { diff --git a/src/daemon/handlers/find.ts b/src/daemon/handlers/find.ts index 42f2c9104..7efc4988d 100644 --- a/src/daemon/handlers/find.ts +++ b/src/daemon/handlers/find.ts @@ -6,7 +6,11 @@ import type { DaemonRequest, DaemonResponse } from '../types.ts'; import { SessionStore } from '../session-store.ts'; import { contextFromFlags } from '../context.ts'; import { ensureDeviceReady } from '../device-ready.ts'; -import { extractNodeText, findNearestHittableAncestor } from '../snapshot-processing.ts'; +import { extractNodeText } from '../snapshot-processing.ts'; +import { + resolveActionableTouchNode, + resolveActionableTouchResolution, +} from '../../commands/interaction-targeting.ts'; import { readTextForNode } from './interaction-read.ts'; import { captureSnapshot } from './snapshot-capture.ts'; import { setSessionSnapshot } from '../session-snapshot.ts'; @@ -37,6 +41,10 @@ type ResolvedMatch = { actionFlags: Record; }; +type FindMatchResult = + | { ok: true; node: SnapshotState['nodes'][number] } + | { ok: false; response: DaemonResponse }; + export async function handleFindCommands(params: { req: DaemonRequest; sessionName: string; @@ -67,8 +75,7 @@ export async function handleFindCommands(params: { }); if (runtimeResponse) return runtimeResponse; const session = sessionStore.get(sessionName); - const isReadOnly = - action === 'exists' || action === 'wait' || action === 'get_text' || action === 'get_attrs'; + const isReadOnly = isReadOnlyFindAction(action); if (!session && !isReadOnly) { return errorResponse('SESSION_NOT_FOUND', 'No active session. Run open first.'); } @@ -76,9 +83,10 @@ export async function handleFindCommands(params: { if (!session) { await ensureDeviceReady(device); } - const scope = shouldScopeFind(locator) ? query : undefined; - const requiresRect = - action === 'click' || action === 'focus' || action === 'fill' || action === 'type'; + const requiresRect = findActionRequiresRect(action); + // Interaction targets need the full compact tree so duplicate labels can be + // resolved against viewport visibility before an off-screen subtree wins. + const scope = shouldScopeFind(locator) && !requiresRect ? query : undefined; const interactiveOnly = requiresRect; let lastSnapshotAt = 0; let lastNodes: SnapshotState['nodes'] | null = null; @@ -134,29 +142,16 @@ export async function handleFindCommands(params: { } const { nodes } = await fetchNodes(); - const bestMatches = findBestMatchesByLocator(nodes, locator, query, { - requireRect: requiresRect, + const matchResult = resolveFindMatch({ + nodes, + locator, + query, + requiresRect, + flags: req.flags, }); - - if (requiresRect && bestMatches.matches.length > 1) { - if (req.flags?.findFirst) { - bestMatches.matches = [bestMatches.matches[0]]; - } else if (req.flags?.findLast) { - bestMatches.matches = [bestMatches.matches[bestMatches.matches.length - 1]]; - } else { - return buildAmbiguousMatchError(bestMatches.matches, locator, query); - } - } - - const node = bestMatches.matches[0] ?? null; - if (!node) { - return errorResponse('COMMAND_FAILED', 'find did not match any element'); - } - - const resolvedNode = - action === 'click' || action === 'focus' || action === 'fill' || action === 'type' - ? (findNearestHittableAncestor(nodes, node) ?? node) - : node; + if (!matchResult.ok) return matchResult.response; + const node = matchResult.node; + const resolvedNode = requiresRect ? resolveInteractiveMatchNode(nodes, node) : node; const ref = `@${resolvedNode.ref}`; const actionFlags = { ...(req.flags ?? {}), noRecord: true }; const match: ResolvedMatch = { node, resolvedNode, ref, nodes, actionFlags }; @@ -177,6 +172,128 @@ export async function handleFindCommands(params: { // --- Per-action handlers --- +function isReadOnlyFindAction(action: string): boolean { + return ( + action === 'exists' || action === 'wait' || action === 'get_text' || action === 'get_attrs' + ); +} + +function findActionRequiresRect(action: string): boolean { + return action === 'click' || action === 'focus' || action === 'fill' || action === 'type'; +} + +function resolveFindMatch(params: { + nodes: SnapshotState['nodes']; + locator: FindLocator; + query: string; + requiresRect: boolean; + flags: DaemonRequest['flags']; +}): FindMatchResult { + const { nodes, locator, query, requiresRect, flags } = params; + const bestMatches = findBestMatchesByLocator(nodes, locator, query, { + requireRect: requiresRect, + }); + if (requiresRect) { + bestMatches.matches = preferOnscreenMatches(bestMatches.matches, nodes); + } + + if (requiresRect && bestMatches.matches.length > 1) { + if (flags?.findFirst) { + bestMatches.matches = [bestMatches.matches[0]]; + } else if (flags?.findLast) { + bestMatches.matches = [bestMatches.matches[bestMatches.matches.length - 1]]; + } else { + return { ok: false, response: buildAmbiguousMatchError(bestMatches.matches, locator, query) }; + } + } + + const node = bestMatches.matches[0] ?? null; + if (!node) { + return { + ok: false, + response: errorResponse('COMMAND_FAILED', 'find did not match any element'), + }; + } + return { ok: true, node }; +} + +function preferOnscreenMatches( + matches: SnapshotState['nodes'], + nodes: SnapshotState['nodes'], +): SnapshotState['nodes'] { + const viewport = nodes[0]?.rect; + if (!viewport) return matches; + const onscreen = matches.filter((node) => { + if (!node.rect) return false; + const center = centerOfRect(node.rect); + return ( + center.x >= viewport.x && + center.x <= viewport.x + viewport.width && + center.y >= viewport.y && + center.y <= viewport.y + viewport.height + ); + }); + return rankInteractiveMatches(onscreen.length > 0 ? onscreen : matches, nodes); +} + +function rankInteractiveMatches( + matches: SnapshotState['nodes'], + nodes: SnapshotState['nodes'], +): SnapshotState['nodes'] { + if (matches.length < 2) return matches; + return matches + .map((node, index) => ({ node, index, score: interactiveMatchScore(node, nodes) })) + .sort((left, right) => { + if (right.score !== left.score) return right.score - left.score; + return rectArea(left.node) - rectArea(right.node) || left.index - right.index; + }) + .map((entry) => entry.node); +} + +function interactiveMatchScore( + node: SnapshotState['nodes'][number], + nodes: SnapshotState['nodes'], +): number { + const resolution = resolveActionableTouchResolution(nodes, node); + if (resolution.reason === 'semantic-target' && resolution.node.rect) return 4; + if (resolution.reason === 'same-rect-descendant' && resolution.node.rect) return 4; + if ( + resolution.reason === 'hittable-ancestor' && + resolution.node.rect && + !isRootInteractionContainer(resolution.node, nodes[0]) + ) { + return 2; + } + if (node.hittable && node.rect && !isRootInteractionContainer(node, nodes[0])) return 3; + return node.rect ? 1 : 0; +} + +function rectArea(node: SnapshotState['nodes'][number]): number { + return node.rect ? node.rect.width * node.rect.height : Number.POSITIVE_INFINITY; +} + +function resolveInteractiveMatchNode( + nodes: SnapshotState['nodes'], + node: SnapshotState['nodes'][number], +): SnapshotState['nodes'][number] { + return resolveActionableTouchNode(nodes, node); +} + +function isRootInteractionContainer( + node: SnapshotState['nodes'][number], + root: SnapshotState['nodes'][number] | undefined, +): boolean { + if (!root?.rect || !node.rect) return false; + const type = node.type?.toLowerCase() ?? ''; + if (!type.includes('application') && !type.includes('window')) return false; + return ( + node.rect.x === root.rect.x && + node.rect.y === root.rect.y && + node.rect.width === root.rect.width && + node.rect.height === root.rect.height + ); +} + async function handleFindWait( ctx: FindContext, fetchNodes: () => Promise<{ nodes: SnapshotState['nodes'] }>, @@ -266,7 +383,11 @@ async function handleFindClick(ctx: FindContext, match: ResolvedMatch): Promise< flags: match.actionFlags, }); if (!response.ok) return response; - const matchCoords = match.resolvedNode.rect ? centerOfRect(match.resolvedNode.rect) : null; + const matchCoords = match.resolvedNode.rect + ? centerOfRect(match.resolvedNode.rect) + : match.node.rect + ? centerOfRect(match.node.rect) + : null; const matchData: Record = { ref: match.ref, locator, query }; if (matchCoords) { matchData.x = matchCoords.x; @@ -312,7 +433,35 @@ async function handleFindFill( } async function handleFindFocus(ctx: FindContext, match: ResolvedMatch): Promise { - const { req, sessionStore, session, device, command, logPath } = ctx; + const response = await dispatchFocusForFindMatch(ctx, match); + if (!response.ok) return response; + recordFindAction(ctx, match, 'focus'); + return response; +} + +async function handleFindType( + ctx: FindContext, + match: ResolvedMatch, + value: string | undefined, +): Promise { + const { req, device, logPath, session } = ctx; + if (!value) { + return errorResponse('INVALID_ARGS', 'find type requires text'); + } + const focusResponse = await dispatchFocusForFindMatch(ctx, match); + if (!focusResponse.ok) return focusResponse; + const response = await dispatchCommand(device, 'type', [value], req.flags?.out, { + ...contextFromFlags(logPath, req.flags, session?.appBundleId, session?.trace?.outPath), + }); + recordFindAction(ctx, match, 'type'); + return { ok: true, data: response ?? { ref: match.ref } }; +} + +async function dispatchFocusForFindMatch( + ctx: FindContext, + match: ResolvedMatch, +): Promise { + const { req, device, logPath, session } = ctx; const coords = match.node.rect ? centerOfRect(match.node.rect) : null; if (!coords) { return errorResponse('COMMAND_FAILED', 'matched element has no bounds'); @@ -326,45 +475,19 @@ async function handleFindFocus(ctx: FindContext, match: ResolvedMatch): Promise< ...contextFromFlags(logPath, req.flags, session?.appBundleId, session?.trace?.outPath), }, ); - if (session) { - sessionStore.recordAction(session, { - command, - positionals: req.positionals ?? [], - flags: req.flags ?? {}, - result: { ref: match.ref, action: 'focus' }, - }); - } return { ok: true, data: response ?? { ref: match.ref } }; } -async function handleFindType( - ctx: FindContext, - match: ResolvedMatch, - value: string | undefined, -): Promise { - const { req, sessionStore, session, device, command, logPath } = ctx; - if (!value) { - return errorResponse('INVALID_ARGS', 'find type requires text'); - } - const coords = match.node.rect ? centerOfRect(match.node.rect) : null; - if (!coords) { - return errorResponse('COMMAND_FAILED', 'matched element has no bounds'); - } - await dispatchCommand(device, 'focus', [String(coords.x), String(coords.y)], req.flags?.out, { - ...contextFromFlags(logPath, req.flags, session?.appBundleId, session?.trace?.outPath), - }); - const response = await dispatchCommand(device, 'type', [value], req.flags?.out, { - ...contextFromFlags(logPath, req.flags, session?.appBundleId, session?.trace?.outPath), - }); +function recordFindAction(ctx: FindContext, match: ResolvedMatch, action: string): void { + const { req, sessionStore, session, command } = ctx; if (session) { sessionStore.recordAction(session, { command, positionals: req.positionals ?? [], flags: req.flags ?? {}, - result: { ref: match.ref, action: 'type' }, + result: { ref: match.ref, action }, }); } - return { ok: true, data: response ?? { ref: match.ref } }; } // --- Helpers --- diff --git a/src/daemon/handlers/interaction-touch-targets.ts b/src/daemon/handlers/interaction-touch-targets.ts index 264302ef4..d682531c0 100644 --- a/src/daemon/handlers/interaction-touch-targets.ts +++ b/src/daemon/handlers/interaction-touch-targets.ts @@ -110,8 +110,7 @@ export function parseFillTarget(positionals: string[]): ParsedFillTarget { ), }; } - const text = parsed.text.trim(); - if (!text) { + if (!parsed.text.trim()) { return { ok: false, response: errorResponse('INVALID_ARGS', 'fill requires text after selector'), @@ -120,7 +119,7 @@ export function parseFillTarget(positionals: string[]): ParsedFillTarget { return { ok: true, target: { kind: 'selector', selector: parsed.target.selector }, - text, + text: parsed.text, }; } diff --git a/src/daemon/handlers/interaction-touch.ts b/src/daemon/handlers/interaction-touch.ts index 9c46354ff..064b95d75 100644 --- a/src/daemon/handlers/interaction-touch.ts +++ b/src/daemon/handlers/interaction-touch.ts @@ -257,7 +257,14 @@ function readDirectIosSelectorTapTarget(params: { if (commandLabel !== 'click') return null; if (target.kind !== 'selector') return null; if (hasNonDefaultClickOptions(flags)) return null; - return readSimpleIosSelectorTarget({ session, selectorExpression: target.selector }); + const selector = readSimpleIosSelectorTarget({ session, selectorExpression: target.selector }); + if (!selector) return null; + return { + ...selector, + ...(flags?.maestro?.allowNonHittableCoordinateFallback + ? { allowNonHittableCoordinateFallback: true } + : {}), + }; } function hasNonDefaultClickOptions(flags: CommandFlags | undefined): boolean { @@ -276,11 +283,44 @@ async function dispatchDirectIosSelectorTap( session: SessionState, selector: DirectIosSelectorTarget, ): Promise { + return await dispatchDirectIosSelectorInteraction({ + params, + session, + selector, + command: 'press', + positionals: [], + extra: { selector: selector.raw }, + fallbackPhase: 'ios_direct_selector_tap_fallback', + }); +} + +async function dispatchDirectIosSelectorInteraction(params: { + params: InteractionHandlerParams; + session: SessionState; + selector: DirectIosSelectorTarget; + command: 'press' | 'fill'; + positionals: string[]; + extra: Record; + fallbackPhase: string; +}): Promise { + const { + params: handlerParams, + session, + selector, + command, + positionals, + extra, + fallbackPhase, + } = params; const actionStartedAt = Date.now(); try { const data = - (await dispatchCommand(session.device, 'press', [], params.req.flags?.out, { - ...params.contextFromFlags(params.req.flags, session.appBundleId, session.trace?.outPath), + (await dispatchCommand(session.device, command, positionals, handlerParams.req.flags?.out, { + ...handlerParams.contextFromFlags( + handlerParams.req.flags, + session.appBundleId, + session.trace?.outPath, + ), directElementSelector: selector, surface: session.surface, })) ?? {}; @@ -291,16 +331,14 @@ async function dispatchDirectIosSelectorTap( fallbackX: point.x, fallbackY: point.y, referenceFrame: readReferenceFrameFromDirectSelectorTapResult(data), - extra: { - selector: selector.raw, - }, + extra, }); return finalizeTouchInteraction({ session, - sessionStore: params.sessionStore, - command: params.req.command, - positionals: params.req.positionals ?? [], - flags: params.req.flags, + sessionStore: handlerParams.sessionStore, + command: handlerParams.req.command, + positionals: handlerParams.req.positionals ?? [], + flags: handlerParams.req.flags, result: responseData, responseData, actionStartedAt, @@ -312,7 +350,7 @@ async function dispatchDirectIosSelectorTap( } emitDiagnostic({ level: 'debug', - phase: 'ios_direct_selector_tap_fallback', + phase: fallbackPhase, data: { selector: selector.raw, error: error instanceof Error ? error.message : String(error), @@ -366,6 +404,20 @@ async function dispatchFillViaRuntime( if (invalidRefFlagsResponse) return invalidRefFlagsResponse; await refreshAndroidRefSnapshotIfFreshnessActive(params, session); } + const directSelector = readDirectIosSelectorFillTarget({ + session, + target: parsedTarget.target, + flags: req.flags, + }); + if (directSelector) { + const directResponse = await dispatchDirectIosSelectorFill( + params, + session, + directSelector, + parsedTarget.text, + ); + if (directResponse) return directResponse; + } return await dispatchRuntimeInteraction(params, { run: async (runtime) => @@ -407,6 +459,40 @@ async function dispatchFillViaRuntime( }); } +function readDirectIosSelectorFillTarget(params: { + session: SessionState; + target: InteractionTarget; + flags: CommandFlags | undefined; +}): DirectIosSelectorTarget | null { + const { session, target, flags } = params; + if (target.kind !== 'selector') return null; + const selector = readSimpleIosSelectorTarget({ session, selectorExpression: target.selector }); + if (!selector) return null; + return { + ...selector, + ...(flags?.maestro?.allowNonHittableCoordinateFallback + ? { allowNonHittableCoordinateFallback: true } + : {}), + }; +} + +async function dispatchDirectIosSelectorFill( + params: InteractionHandlerParams, + session: SessionState, + selector: DirectIosSelectorTarget, + text: string, +): Promise { + return await dispatchDirectIosSelectorInteraction({ + params, + session, + selector, + command: 'fill', + positionals: [text], + extra: { selector: selector.raw, text }, + fallbackPhase: 'ios_direct_selector_fill_fallback', + }); +} + async function dispatchRuntimeInteraction< TResult extends PressCommandResult | FillCommandResult | LongPressCommandResult, >( diff --git a/src/daemon/handlers/session-replay-action-runtime.ts b/src/daemon/handlers/session-replay-action-runtime.ts new file mode 100644 index 000000000..6ad7e658e --- /dev/null +++ b/src/daemon/handlers/session-replay-action-runtime.ts @@ -0,0 +1,148 @@ +import fs from 'node:fs'; +import type { CommandFlags } from '../../core/dispatch.ts'; +import { + mergeReplayVarScopeValues, + resolveReplayAction, + type ReplayVarScope, +} from '../../replay/vars.ts'; +import type { DaemonRequest, DaemonResponse, SessionAction } from '../types.ts'; +import { mergeParentFlags } from './handler-utils.ts'; +import { invokeMaestroRuntimeCommand } from './session-replay-maestro-runtime.ts'; + +type ReplayBaseRequest = Omit; + +type ReplayActionInvoker = (params: { + action: SessionAction; + line: number; + step: number; +}) => Promise; + +export async function invokeReplayAction(params: { + req: DaemonRequest; + sessionName: string; + action: SessionAction; + scope: ReplayVarScope; + filePath: string; + line: number; + step: number; + tracePath?: string; + invoke: (req: DaemonRequest) => Promise; +}): Promise { + const { req, sessionName, action, scope, filePath, line, step, tracePath, invoke } = params; + const resolved = resolveReplayAction(action, scope, { file: filePath, line }); + const invokeNestedReplayAction: ReplayActionInvoker = (nested) => + invokeReplayAction({ + req, + sessionName, + action: nested.action, + scope, + filePath, + line: nested.line, + step: nested.step, + tracePath, + invoke, + }); + const startedAt = Date.now(); + appendReplayTraceEvent(tracePath, { + type: 'replay_action_start', + ts: new Date(startedAt).toISOString(), + replayPath: filePath, + line, + step, + command: resolved.command, + positionals: resolved.positionals ?? [], + }); + + const response = await invokeResolvedReplayAction({ + req, + sessionName, + resolved, + scope, + line, + step, + invoke, + invokeReplayAction: invokeNestedReplayAction, + }); + + const finishedAt = Date.now(); + appendReplayTraceEvent(tracePath, { + type: 'replay_action_stop', + ts: new Date(finishedAt).toISOString(), + replayPath: filePath, + line, + step, + command: resolved.command, + ok: response.ok, + durationMs: finishedAt - startedAt, + errorCode: response.ok ? undefined : response.error.code, + }); + return response; +} + +async function invokeResolvedReplayAction(params: { + req: DaemonRequest; + sessionName: string; + resolved: SessionAction; + scope: ReplayVarScope; + line: number; + step: number; + invoke: (req: DaemonRequest) => Promise; + invokeReplayAction: ReplayActionInvoker; +}): Promise { + const { req, sessionName, resolved, scope, line, step, invoke, invokeReplayAction } = params; + const flags = buildReplayActionFlags(req.flags, resolved.flags); + const baseReq: ReplayBaseRequest = { + token: req.token, + session: sessionName, + flags, + runtime: resolved.runtime, + meta: req.meta, + }; + const response = + (await invokeMaestroRuntimeCommand({ + command: resolved.command, + baseReq, + positionals: resolved.positionals ?? [], + batchSteps: resolved.flags?.batchSteps, + scope, + line, + step, + invoke, + invokeReplayAction, + })) ?? + (await invoke({ + ...baseReq, + command: resolved.command, + positionals: resolved.positionals ?? [], + })); + if (response.ok) { + const outputEnv = readReplayOutputEnv(response.data); + if (outputEnv) mergeReplayVarScopeValues(scope, outputEnv); + } + return response; +} + +function readReplayOutputEnv(data: unknown): Record | null { + if (!data || typeof data !== 'object') return null; + const raw = (data as { outputEnv?: unknown }).outputEnv; + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return null; + const entries = Object.entries(raw).filter( + (entry): entry is [string, string] => typeof entry[1] === 'string', + ); + return entries.length > 0 ? Object.fromEntries(entries) : null; +} + +function appendReplayTraceEvent( + tracePath: string | undefined, + event: Record, +): void { + if (!tracePath) return; + fs.appendFileSync(tracePath, `${JSON.stringify(event)}\n`); +} + +export function buildReplayActionFlags( + parentFlags: CommandFlags | undefined, + actionFlags: SessionAction['flags'] | undefined, +): CommandFlags { + return mergeParentFlags(parentFlags, { ...(actionFlags ?? {}) }); +} diff --git a/src/daemon/handlers/session-replay-maestro-runtime.ts b/src/daemon/handlers/session-replay-maestro-runtime.ts new file mode 100644 index 000000000..8ec68ae15 --- /dev/null +++ b/src/daemon/handlers/session-replay-maestro-runtime.ts @@ -0,0 +1,976 @@ +import { type CommandFlags } from '../../core/dispatch.ts'; +import { MAESTRO_RUNTIME_COMMAND } from '../../compat/maestro/runtime-commands.ts'; +import { executeRunScriptFile } from '../../compat/maestro/run-script.ts'; +import type { Platform } from '../../utils/device.ts'; +import { type Rect, type SnapshotNode, type SnapshotState } from '../../utils/snapshot.ts'; +import { sleep } from '../../utils/timeouts.ts'; +import { asAppError } from '../../utils/errors.ts'; +import type { ReplayVarScope } from '../../replay/vars.ts'; +import { parseSelectorChain } from '../selectors.ts'; +import { matchesSelector } from '../selectors-match.ts'; +import { getSnapshotReferenceFrame, type TouchReferenceFrame } from '../touch-reference-frame.ts'; +import type { DaemonRequest, DaemonResponse, SessionAction } from '../types.ts'; +import { errorResponse } from './response.ts'; + +// Keep Maestro timing and target-selection heuristics behind one policy so +// generic Agent Device command behavior does not inherit compatibility rules. +const MAESTRO_REPLAY_POLICY = { + animationPollMs: 250, + scrollUntilVisibleProbeMs: 500, + tapOnRetryMs: 250, + tapOnTimeoutMs: 30000, + optionalTapOnTimeoutMs: 3000, + swipe: { + screenRatio: 0.35, + minDistancePx: 120, + maxDistancePx: 360, + marginPx: 8, + }, + largeTextContainerBias: { + minWidth: 120, + minHeight: 70, + width: 168, + height: 48, + }, +} as const; + +const MAESTRO_TAP_TARGET_TYPE_RANK = new Map([ + ['button', 0], + ['link', 0], + ['textfield', 0], + ['textview', 0], + ['searchfield', 0], + ['switch', 0], + ['slider', 0], + ['cell', 1], + ['statictext', 2], +]); + +type ReplayBaseRequest = Omit; + +type MaestroReplayInvoker = (params: { + action: SessionAction; + line: number; + step: number; +}) => Promise; + +type MaestroRuntimeInvoke = (req: DaemonRequest) => Promise; +type FailedDaemonResponse = Extract; + +type MaestroScrollUntilVisibleParams = { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: MaestroRuntimeInvoke; +}; + +type MaestroTapOnParams = { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: MaestroRuntimeInvoke; +}; + +type MaestroTapOnOptions = { + childOf?: string; + index?: number; +}; + +type MaestroRunFlowWhenCondition = + | { ok: true; mode: string; predicate: string; selector: string } + | { ok: false; response: DaemonResponse }; + +type MaestroSnapshotTarget = { + node: SnapshotNode; + rect: Rect; + frame?: TouchReferenceFrame; +}; + +export async function invokeMaestroRuntimeCommand(params: { + command: string; + baseReq: ReplayBaseRequest; + positionals: string[]; + batchSteps: CommandFlags['batchSteps'] | undefined; + scope: ReplayVarScope; + line: number; + step: number; + invoke: (req: DaemonRequest) => Promise; + invokeReplayAction: MaestroReplayInvoker; +}): Promise { + switch (params.command) { + case MAESTRO_RUNTIME_COMMAND.assertNotVisible: + return await invokeMaestroAssertNotVisible(params); + case MAESTRO_RUNTIME_COMMAND.pressEnter: + return await invokeMaestroPressEnter(params); + case MAESTRO_RUNTIME_COMMAND.waitForAnimationToEnd: + return await invokeMaestroWaitForAnimationToEnd(params); + case MAESTRO_RUNTIME_COMMAND.scrollUntilVisible: + return await invokeMaestroScrollUntilVisible(params); + case MAESTRO_RUNTIME_COMMAND.swipeOn: + return await invokeMaestroSwipeOn(params); + case MAESTRO_RUNTIME_COMMAND.tapOn: + return await invokeMaestroTapOn(params); + case MAESTRO_RUNTIME_COMMAND.tapPointPercent: + return await invokeMaestroTapPointPercent(params); + case MAESTRO_RUNTIME_COMMAND.runFlowWhen: + return await invokeMaestroRunFlowWhen(params); + case MAESTRO_RUNTIME_COMMAND.runScript: + return invokeMaestroRunScript(params); + default: + return undefined; + } +} + +async function invokeMaestroPressEnter(params: { + baseReq: ReplayBaseRequest; + invoke: MaestroRuntimeInvoke; +}): Promise { + const keyboardResponse = await params.invoke({ + ...params.baseReq, + command: 'keyboard', + positionals: ['enter'], + }); + if (keyboardResponse.ok) return keyboardResponse; + + return await params.invoke({ + ...params.baseReq, + command: 'type', + positionals: ['\n'], + }); +} + +async function invokeMaestroAssertNotVisible(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: MaestroRuntimeInvoke; +}): Promise { + const [selector] = params.positionals; + if (!selector) { + return errorResponse('INVALID_ARGS', 'assertNotVisible requires a selector.'); + } + const response = await params.invoke({ + ...params.baseReq, + command: 'is', + positionals: ['visible', selector], + flags: { ...params.baseReq.flags, noRecord: true }, + }); + if (!response.ok) { + return { ok: true, data: { pass: true, selector, absent: true } }; + } + if (response.data?.pass === false) { + return { ok: true, data: { pass: true, selector } }; + } + return errorResponse('COMMAND_FAILED', `Expected not visible but matched: ${selector}`); +} + +async function invokeMaestroWaitForAnimationToEnd(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: MaestroRuntimeInvoke; +}): Promise { + const timeoutMs = Number(params.positionals[0] ?? 15000); + if (!Number.isFinite(timeoutMs) || timeoutMs < 0) { + return errorResponse('INVALID_ARGS', 'waitForAnimationToEnd timeout must be a number.'); + } + const startedAt = Date.now(); + let previousSignature: string | undefined; + let lastResponse: DaemonResponse | undefined; + + while (Date.now() - startedAt < timeoutMs) { + const response = await captureMaestroRawSnapshot(params); + const poll = readAnimationPollResult(response, previousSignature, timeoutMs); + if (poll.done) return poll.response; + previousSignature = poll.signature ?? previousSignature; + lastResponse = response; + await sleep(MAESTRO_REPLAY_POLICY.animationPollMs); + } + + return lastResponse?.ok === false + ? lastResponse + : { ok: true, data: { stable: false, timeoutMs } }; +} + +function readAnimationPollResult( + response: DaemonResponse, + previousSignature: string | undefined, + timeoutMs: number, +): { done: true; response: DaemonResponse } | { done: false; signature?: string } { + const signature = readSnapshotStabilitySignature(response); + if (!response.ok) return { done: false }; + if (!signature) return { done: true, response }; + if (previousSignature === signature) { + return { done: true, response: { ok: true, data: { stable: true, timeoutMs } } }; + } + return { done: false, signature }; +} + +async function captureMaestroRawSnapshot(params: { + baseReq: ReplayBaseRequest; + invoke: MaestroRuntimeInvoke; +}): Promise { + return await params.invoke({ + ...params.baseReq, + command: 'snapshot', + positionals: [], + flags: { + ...params.baseReq.flags, + noRecord: true, + snapshotRaw: true, + snapshotForceFull: true, + }, + }); +} + +function readSnapshotStabilitySignature(response: DaemonResponse): string | null { + if (!response.ok) return null; + const snapshot = readSnapshotState(response.data); + return snapshot ? snapshotStabilitySignature(snapshot) : null; +} + +async function invokeMaestroScrollUntilVisible( + params: MaestroScrollUntilVisibleParams, +): Promise { + const [selector, timeoutValue = '5000', direction = 'down'] = params.positionals; + if (!selector) { + return errorResponse('INVALID_ARGS', 'scrollUntilVisible requires a selector.'); + } + const timeoutMs = Number(timeoutValue); + if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { + return errorResponse('INVALID_ARGS', 'scrollUntilVisible timeout must be a positive number.'); + } + const fuzzyTextQuery = extractMaestroVisibleTextQuery(selector); + const attempts = Math.max( + 1, + Math.ceil(timeoutMs / MAESTRO_REPLAY_POLICY.scrollUntilVisibleProbeMs), + ); + let lastWaitResponse: FailedDaemonResponse | null = null; + + for (let index = 0; index < attempts; index += 1) { + const probeResponse = await probeMaestroScrollVisibility( + params, + selector, + fuzzyTextQuery, + scrollProbeMs(timeoutMs, index), + ); + if (probeResponse.ok) return probeResponse; + lastWaitResponse = probeResponse; + + if (index === attempts - 1) break; + + const scrollResponse = await params.invoke({ + ...params.baseReq, + command: 'scroll', + positionals: [direction], + }); + if (!scrollResponse.ok) return scrollResponse; + } + + return withMaestroScrollTimeoutContext(lastWaitResponse, selector, timeoutMs); +} + +async function probeMaestroScrollVisibility( + params: MaestroScrollUntilVisibleParams, + selector: string, + fuzzyTextQuery: string | null, + probeMs: number, +): Promise { + const waitResponse = await params.invoke({ + ...params.baseReq, + command: 'wait', + positionals: [selector, String(probeMs)], + }); + if (waitResponse.ok || !fuzzyTextQuery) return waitResponse; + + const fuzzyResponse = await params.invoke({ + ...params.baseReq, + command: 'find', + positionals: [fuzzyTextQuery, 'wait', String(probeMs)], + }); + return fuzzyResponse; +} + +function scrollProbeMs(timeoutMs: number, index: number): number { + return Math.min( + MAESTRO_REPLAY_POLICY.scrollUntilVisibleProbeMs, + Math.max(1, timeoutMs - index * MAESTRO_REPLAY_POLICY.scrollUntilVisibleProbeMs), + ); +} + +async function invokeMaestroTapPointPercent(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: (req: DaemonRequest) => Promise; +}): Promise { + const [xValue, yValue] = params.positionals; + const xPercent = Number(xValue); + const yPercent = Number(yValue); + if (!Number.isFinite(xPercent) || !Number.isFinite(yPercent)) { + return errorResponse('INVALID_ARGS', 'tapOn percentage point requires numeric x/y values.'); + } + + const snapshotResponse = await params.invoke({ + ...params.baseReq, + command: 'snapshot', + positionals: [], + flags: { + ...params.baseReq.flags, + noRecord: true, + snapshotRaw: true, + snapshotForceFull: true, + }, + }); + if (!snapshotResponse.ok) return snapshotResponse; + + const snapshot = readSnapshotState(snapshotResponse.data); + if (!snapshot) { + return errorResponse( + 'COMMAND_FAILED', + 'Unable to read snapshot data for Maestro percentage point tap.', + ); + } + + const frame = getSnapshotReferenceFrame(snapshot); + if (!frame) { + return errorResponse( + 'COMMAND_FAILED', + 'Unable to resolve screen size for Maestro percentage point tap.', + ); + } + + return await params.invoke({ + ...params.baseReq, + command: 'click', + positionals: [ + String(Math.round((frame.referenceWidth * xPercent) / 100)), + String(Math.round((frame.referenceHeight * yPercent) / 100)), + ], + }); +} + +function readSnapshotState(data: unknown): SnapshotState | undefined { + if ( + typeof data === 'object' && + data !== null && + Array.isArray((data as { nodes?: unknown }).nodes) + ) { + return data as SnapshotState; + } + return undefined; +} + +function snapshotStabilitySignature(snapshot: SnapshotState): string { + return JSON.stringify( + snapshot.nodes.map((node) => ({ + index: node.index, + parentIndex: node.parentIndex, + type: node.type, + identifier: node.identifier, + label: node.label, + value: node.value, + rect: node.rect + ? { + x: Math.round(node.rect.x), + y: Math.round(node.rect.y), + width: Math.round(node.rect.width), + height: Math.round(node.rect.height), + } + : undefined, + })), + ); +} + +async function invokeMaestroTapOn(params: MaestroTapOnParams): Promise { + const [selector, rawOptions] = params.positionals; + if (!selector) { + return errorResponse('INVALID_ARGS', 'tapOn requires a selector.'); + } + const options = readMaestroTapOnOptions(rawOptions); + if (!options.ok) return options.response; + const startedAt = Date.now(); + const fuzzyTextQuery = extractMaestroVisibleTextQuery(selector); + const timeoutMs = maestroTapOnTimeoutMs(params); + let lastResponse: DaemonResponse | undefined; + while (Date.now() - startedAt < timeoutMs) { + const attempt = await attemptMaestroTapOn( + params, + selector, + options.value ?? {}, + fuzzyTextQuery, + ); + if (!attempt.retry) return attempt.response; + lastResponse = attempt.response; + await sleep(MAESTRO_REPLAY_POLICY.tapOnRetryMs); + } + + return maestroTapOnTimeoutResponse(params, selector, lastResponse); +} + +function maestroTapOnTimeoutMs(params: MaestroTapOnParams): number { + return params.baseReq.flags?.maestro?.optional === true + ? MAESTRO_REPLAY_POLICY.optionalTapOnTimeoutMs + : MAESTRO_REPLAY_POLICY.tapOnTimeoutMs; +} + +function maestroTapOnTimeoutResponse( + params: MaestroTapOnParams, + selector: string, + lastResponse: DaemonResponse | undefined, +): DaemonResponse { + if (params.baseReq.flags?.maestro?.optional === true) { + return { ok: true, data: { skipped: true, optional: true, selector } }; + } + return ( + lastResponse ?? errorResponse('COMMAND_FAILED', `tapOn timed out for selector: ${selector}`) + ); +} + +async function attemptMaestroTapOn( + params: MaestroTapOnParams, + selector: string, + options: MaestroTapOnOptions, + fuzzyTextQuery: string | null, +): Promise< + { retry: false; response: DaemonResponse } | { retry: true; response: FailedDaemonResponse } +> { + const attempt = await invokeMaestroSnapshotTapOn(params, selector, options); + if (attempt.ok) return { retry: false, response: attempt }; + if (!fuzzyTextQuery) return { retry: true, response: attempt }; + return await invokeMaestroFuzzyTapOn(params, fuzzyTextQuery); +} + +async function invokeMaestroSnapshotTapOn( + params: MaestroTapOnParams, + selector: string, + options: MaestroTapOnOptions, +): Promise { + const target = await resolveMaestroSnapshotTarget(params, selector, options, 'tapOn'); + if (!target.ok) return target.response; + const point = pointForMaestroTapOnTarget(target.target, selector); + return await params.invoke({ + ...params.baseReq, + command: 'click', + positionals: [String(point.x), String(point.y)], + }); +} + +async function invokeMaestroSwipeOn(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: MaestroRuntimeInvoke; +}): Promise { + const [selector, direction = 'up', durationMs] = params.positionals; + if (!selector) return errorResponse('INVALID_ARGS', 'swipe.label requires a label selector.'); + const target = await resolveMaestroSnapshotTarget(params, selector, {}, 'swipe.label'); + if (!target.ok) return target.response; + const swipe = swipeCoordinatesFromTarget(target.target, direction); + if (!swipe.ok) return swipe.response; + return await params.invoke({ + ...params.baseReq, + command: 'swipe', + positionals: [ + String(swipe.start.x), + String(swipe.start.y), + String(swipe.end.x), + String(swipe.end.y), + ...(durationMs ? [durationMs] : []), + ], + }); +} + +async function invokeMaestroFuzzyTapOn( + params: MaestroTapOnParams, + query: string, +): Promise< + { retry: false; response: DaemonResponse } | { retry: true; response: FailedDaemonResponse } +> { + const findResponse = await params.invoke({ + ...params.baseReq, + command: 'find', + positionals: [query, 'click'], + flags: { + ...params.baseReq.flags, + findFirst: true, + }, + }); + if (findResponse.ok) return { retry: false, response: findResponse }; + return { retry: true, response: findResponse }; +} + +async function resolveMaestroSnapshotTarget( + params: { + baseReq: ReplayBaseRequest; + invoke: MaestroRuntimeInvoke; + }, + selector: string, + options: MaestroTapOnOptions, + commandLabel: string, +): Promise<{ ok: true; target: MaestroSnapshotTarget } | { ok: false; response: DaemonResponse }> { + const snapshotResponse = await params.invoke({ + ...params.baseReq, + command: 'snapshot', + positionals: [], + flags: { + ...params.baseReq.flags, + noRecord: true, + snapshotRaw: true, + snapshotForceFull: true, + }, + }); + if (!snapshotResponse.ok) return { ok: false, response: snapshotResponse }; + + const snapshot = readSnapshotState(snapshotResponse.data); + if (!snapshot) { + return { + ok: false, + response: errorResponse( + 'COMMAND_FAILED', + `Unable to read snapshot data for ${commandLabel}.`, + ), + }; + } + + const frame = getSnapshotReferenceFrame(snapshot); + const resolution = resolveMaestroNodeFromSnapshot( + snapshot, + selector, + options, + readMaestroSelectorPlatform(params.baseReq.flags), + frame, + ); + if (!resolution.ok) { + return { + ok: false, + response: errorResponse('ELEMENT_NOT_FOUND', resolution.message), + }; + } + return { + ok: true, + target: { + node: resolution.node, + rect: resolution.rect, + frame, + }, + }; +} + +function resolveMaestroNodeFromSnapshot( + snapshot: SnapshotState, + selector: string, + options: MaestroTapOnOptions, + platform: Platform, + frame: TouchReferenceFrame | undefined, +): { ok: true; node: SnapshotNode; rect: Rect } | { ok: false; message: string } { + let matches = findMaestroSelectorMatches(snapshot, selector, platform); + if (options.childOf) { + const parents = findMaestroSelectorMatches(snapshot, options.childOf, platform); + if (parents.length === 0) { + return { ok: false, message: `Maestro childOf parent did not match: ${options.childOf}` }; + } + matches = matches.filter((node) => + parents.some((parent) => isDescendantOfSnapshotNode(snapshot.nodes, node, parent)), + ); + } + + const target = selectMaestroSnapshotMatch( + snapshot.nodes, + matches, + options.index, + extractMaestroVisibleTextQuery(selector) !== null, + frame, + ); + if (!target) { + const index = options.index ?? 0; + return { + ok: false, + message: `Maestro selector did not match index ${index}: ${selector}`, + }; + } + return { ok: true, node: target.node, rect: target.rect }; +} + +function findMaestroSelectorMatches( + snapshot: SnapshotState, + selectorExpression: string, + platform: Platform, +): SnapshotNode[] { + const chain = parseSelectorChain(selectorExpression); + for (const selector of chain.selectors) { + const matches = snapshot.nodes.filter((node) => matchesSelector(node, selector, platform)); + if (matches.length > 0) return matches; + } + return []; +} + +function resolveNodeRect(nodes: SnapshotState['nodes'], node: SnapshotNode): Rect | null { + if (node.rect && node.rect.width > 0 && node.rect.height > 0) return node.rect; + return ( + findSnapshotAncestor(nodes, node, (ancestor) => + ancestor.rect && ancestor.rect.width > 0 && ancestor.rect.height > 0 ? ancestor : null, + )?.rect ?? null + ); +} + +function selectMaestroSnapshotMatch( + nodes: SnapshotState['nodes'], + matches: SnapshotNode[], + index: number | undefined, + preferOnScreen: boolean, + frame: TouchReferenceFrame | undefined, +): { node: SnapshotNode; rect: Rect } | null { + const resolved = matches + .map((node) => { + const rect = resolveNodeRect(nodes, node); + return rect ? { node, rect } : null; + }) + .filter((candidate): candidate is { node: SnapshotNode; rect: Rect } => Boolean(candidate)); + const candidates = + preferOnScreen && index === undefined ? preferOnScreenMatches(resolved, frame) : resolved; + if (index !== undefined) return candidates[index] ?? null; + return candidates.sort(compareMaestroSnapshotMatches)[0] ?? null; +} + +function preferOnScreenMatches( + matches: { node: SnapshotNode; rect: Rect }[], + frame: TouchReferenceFrame | undefined, +): { node: SnapshotNode; rect: Rect }[] { + const onScreen = matches.filter((match) => isRectOnScreen(match.rect, frame)); + return onScreen.length > 0 ? onScreen : matches; +} + +function isRectOnScreen(rect: Rect, frame: TouchReferenceFrame | undefined): boolean { + const maxX = frame?.referenceWidth ?? Number.POSITIVE_INFINITY; + const maxY = frame?.referenceHeight ?? Number.POSITIVE_INFINITY; + return rect.x < maxX && rect.y < maxY && rect.x + rect.width > 0 && rect.y + rect.height > 0; +} + +function compareMaestroSnapshotMatches( + left: { node: SnapshotNode; rect: Rect }, + right: { node: SnapshotNode; rect: Rect }, +): number { + const typeRank = maestroTapTargetTypeRank(left.node) - maestroTapTargetTypeRank(right.node); + if (typeRank !== 0) return typeRank; + + const areaRank = left.rect.width * left.rect.height - right.rect.width * right.rect.height; + if (areaRank !== 0) return areaRank; + + return (right.node.depth ?? 0) - (left.node.depth ?? 0); +} + +function maestroTapTargetTypeRank(node: SnapshotNode): number { + return MAESTRO_TAP_TARGET_TYPE_RANK.get(node.type?.toLowerCase() ?? '') ?? 3; +} + +function isDescendantOfSnapshotNode( + nodes: SnapshotState['nodes'], + node: SnapshotNode, + ancestor: SnapshotNode, +): boolean { + return Boolean( + findSnapshotAncestor(nodes, node, (candidate) => + candidate === ancestor || candidate.index === ancestor.index ? candidate : null, + ), + ); +} + +function findSnapshotAncestor( + nodes: SnapshotState['nodes'], + node: SnapshotNode, + resolve: (ancestor: SnapshotNode) => T | null, +): T | null { + let current: SnapshotNode | undefined = node; + const byIndex = new Map(nodes.map((candidate) => [candidate.index, candidate])); + while (typeof current.parentIndex === 'number') { + current = byIndex.get(current.parentIndex) ?? nodes[current.parentIndex]; + if (!current) return null; + const result = resolve(current); + if (result) return result; + } + return null; +} + +function readMaestroTapOnOptions( + rawOptions: string | undefined, +): { ok: true; value: MaestroTapOnOptions | null } | { ok: false; response: DaemonResponse } { + if (!rawOptions) return { ok: true, value: null }; + try { + const value = JSON.parse(rawOptions) as MaestroTapOnOptions; + return { ok: true, value }; + } catch { + return { + ok: false, + response: errorResponse('INVALID_ARGS', 'tapOn runtime options must be valid JSON.'), + }; + } +} + +function readMaestroSelectorPlatform(flags: ReplayBaseRequest['flags']): Platform { + return flags?.platform === 'android' ? 'android' : 'ios'; +} + +function swipeCoordinatesFromTarget( + target: MaestroSnapshotTarget, + direction: string, +): + | { ok: true; start: { x: number; y: number }; end: { x: number; y: number } } + | { ok: false; response: DaemonResponse } { + const center = pointInsideRect(target.rect); + const frame = target.frame; + const horizontalDistance = swipeDistance(frame?.referenceWidth, target.rect.width); + const verticalDistance = swipeDistance(frame?.referenceHeight, target.rect.height); + const margin = MAESTRO_REPLAY_POLICY.swipe.marginPx; + const minX = margin; + const minY = margin; + const maxX = frame ? frame.referenceWidth - margin : center.x + horizontalDistance; + const maxY = frame ? frame.referenceHeight - margin : center.y + verticalDistance; + switch (direction.toLowerCase()) { + case 'up': + return { + ok: true, + start: center, + end: { x: center.x, y: clampCoordinate(center.y - verticalDistance, minY, maxY) }, + }; + case 'down': + return { + ok: true, + start: center, + end: { x: center.x, y: clampCoordinate(center.y + verticalDistance, minY, maxY) }, + }; + case 'left': + return { + ok: true, + start: center, + end: { x: clampCoordinate(center.x - horizontalDistance, minX, maxX), y: center.y }, + }; + case 'right': + return { + ok: true, + start: center, + end: { x: clampCoordinate(center.x + horizontalDistance, minX, maxX), y: center.y }, + }; + default: + return { + ok: false, + response: errorResponse( + 'INVALID_ARGS', + 'swipe.label direction must be up, down, left, or right.', + ), + }; + } +} + +function swipeDistance(frameSize: number | undefined, rectSize: number): number { + const screenRelative = + typeof frameSize === 'number' ? frameSize * MAESTRO_REPLAY_POLICY.swipe.screenRatio : 0; + return Math.round( + Math.min( + MAESTRO_REPLAY_POLICY.swipe.maxDistancePx, + Math.max(MAESTRO_REPLAY_POLICY.swipe.minDistancePx, screenRelative, rectSize * 1.5), + ), + ); +} + +function clampCoordinate(value: number, min: number, max: number): number { + return Math.round(Math.min(max, Math.max(min, value))); +} + +function pointInsideRect(rect: Rect): { x: number; y: number } { + return { + x: interiorCoordinate(rect.x, rect.width), + y: interiorCoordinate(rect.y, rect.height), + }; +} + +function pointForMaestroTapOnTarget( + target: MaestroSnapshotTarget, + selector: string, +): { x: number; y: number } { + if (!shouldBiasMaestroVisibleTextTap(target.node, selector, target.rect)) { + return pointInsideRect(target.rect); + } + return { + x: interiorCoordinate( + target.rect.x, + Math.min(target.rect.width, MAESTRO_REPLAY_POLICY.largeTextContainerBias.width), + ), + y: interiorCoordinate( + target.rect.y, + Math.min(target.rect.height, MAESTRO_REPLAY_POLICY.largeTextContainerBias.height), + ), + }; +} + +function shouldBiasMaestroVisibleTextTap( + node: SnapshotNode, + selector: string, + rect: Rect, +): boolean { + if (!extractMaestroVisibleTextQuery(selector)) return false; + if ( + rect.height < MAESTRO_REPLAY_POLICY.largeTextContainerBias.minHeight || + rect.width < MAESTRO_REPLAY_POLICY.largeTextContainerBias.minWidth + ) { + return false; + } + const type = node.type?.toLowerCase(); + return type === 'cell' || type === 'other' || type === 'scrollview'; +} + +function interiorCoordinate(origin: number, size: number): number { + if (size <= 1) return Math.floor(origin); + const min = Math.ceil(origin); + const max = Math.floor(origin + size - 1); + return clampCoordinate(origin + size / 2, min, max); +} + +function invokeMaestroRunScript(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + scope: ReplayVarScope; +}): DaemonResponse { + const [scriptPath] = params.positionals; + if (!scriptPath) { + return errorResponse('INVALID_ARGS', 'runScript requires a file path.'); + } + try { + const outputEnv = executeRunScriptFile({ + scriptPath, + env: { + ...params.scope.values, + ...(params.baseReq.flags?.maestro?.runScriptEnv ?? {}), + }, + }); + return { ok: true, data: { outputEnv } }; + } catch (error) { + const appError = asAppError(error); + return errorResponse(appError.code, appError.message, appError.details); + } +} + +async function invokeMaestroRunFlowWhen(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + batchSteps: CommandFlags['batchSteps'] | undefined; + line: number; + step: number; + invoke: (req: DaemonRequest) => Promise; + invokeReplayAction: MaestroReplayInvoker; +}): Promise { + const condition = readMaestroRunFlowWhenCondition(params.positionals); + if (!condition.ok) return condition.response; + const conditionResponse = await params.invoke({ + ...params.baseReq, + command: 'is', + positionals: [condition.predicate, condition.selector], + flags: { ...params.baseReq.flags, noRecord: true }, + }); + if (isMaestroWhenConditionMiss(conditionResponse)) { + return { + ok: true, + data: { skipped: true, condition: condition.mode, selector: condition.selector }, + }; + } + if (!conditionResponse.ok) return conditionResponse; + return await invokeMaestroRunFlowWhenSteps(params, condition); +} + +function readMaestroRunFlowWhenCondition(positionals: string[]): MaestroRunFlowWhenCondition { + const [mode, selector] = positionals; + if ((mode !== 'visible' && mode !== 'notVisible') || !selector) { + return { + ok: false, + response: errorResponse( + 'INVALID_ARGS', + 'runFlow.when requires visible/notVisible and a selector.', + ), + }; + } + return { + ok: true, + mode, + predicate: mode === 'visible' ? 'visible' : 'hidden', + selector, + }; +} + +async function invokeMaestroRunFlowWhenSteps( + params: { + batchSteps: CommandFlags['batchSteps'] | undefined; + line: number; + step: number; + invokeReplayAction: MaestroReplayInvoker; + }, + condition: Extract, +): Promise { + const steps = (params.batchSteps ?? []).map(batchStepToSessionAction); + for (const [index, action] of steps.entries()) { + // Preserve stable parent-step ordering for nested runtime commands while + // keeping the substep distinguishable in traces. + const response = await params.invokeReplayAction({ + action, + line: params.line, + step: params.step + index / 1000, + }); + if (!response.ok) return response; + } + + return { + ok: true, + data: { ran: steps.length, condition: condition.mode, selector: condition.selector }, + }; +} + +function isMaestroWhenConditionMiss(response: DaemonResponse): boolean { + if (response.ok) return response.data?.pass === false; + const details = response.error.details; + return ( + details?.command === 'is' && + (details.reason === 'selector_not_found' || details.reason === 'predicate_failed') + ); +} + +function batchStepToSessionAction( + step: NonNullable[number], +): SessionAction { + const action: SessionAction = { + ts: Date.now(), + command: step.command, + positionals: step.positionals ?? [], + flags: step.flags ?? {}, + }; + if (step.runtime && typeof step.runtime === 'object') { + action.runtime = step.runtime as SessionAction['runtime']; + } + return action; +} + +function extractMaestroVisibleTextQuery(selectorExpression: string): string | null { + const chain = parseSelectorChain(selectorExpression); + const terms = chain.selectors.flatMap((selector) => selector.terms); + if (terms.length === 0) return null; + // Mixed selectors may encode more than a visible-text lookup, so they keep + // the exact selector path instead of fuzzy text fallback. + if (!terms.some((term) => term.key === 'label' || term.key === 'text')) return null; + if (!terms.every((term) => ['label', 'text', 'id'].includes(term.key))) return null; + const values = terms.map((term) => (typeof term.value === 'string' ? term.value : '')); + const first = values[0]; + if (!first || !values.every((value) => value === first)) return null; + return first; +} + +function withMaestroScrollTimeoutContext( + response: FailedDaemonResponse | null, + selector: string, + timeoutMs: number, +): DaemonResponse { + if (!response) { + return errorResponse( + 'COMMAND_FAILED', + `scrollUntilVisible timed out after ${timeoutMs}ms for selector: ${selector}`, + ); + } + return { + ok: false, + error: { + ...response.error, + message: `scrollUntilVisible timed out after ${timeoutMs}ms for selector: ${selector}. Last wait: ${response.error.message}`, + }, + }; +} diff --git a/src/daemon/handlers/session-replay-runtime.ts b/src/daemon/handlers/session-replay-runtime.ts index 12ff2cee0..9b830fee0 100644 --- a/src/daemon/handlers/session-replay-runtime.ts +++ b/src/daemon/handlers/session-replay-runtime.ts @@ -8,16 +8,14 @@ import { SessionStore } from '../session-store.ts'; import { type ReplayScriptMetadata, writeReplayScript } from '../../replay/script.ts'; import { healReplayAction } from './session-replay-heal.ts'; import { formatScriptActionSummary } from '../../replay/script-utils.ts'; -import { mergeParentFlags } from './handler-utils.ts'; import { errorResponse } from './response.ts'; +import { invokeReplayAction } from './session-replay-action-runtime.ts'; import { buildReplayVarScope, collectReplayShellEnv, parseReplayCliEnvEntries, readReplayCliEnvEntries, readReplayShellEnvSource, - resolveReplayAction, - type ReplayVarScope, } from '../../replay/vars.ts'; // fallow-ignore-next-line complexity @@ -157,61 +155,6 @@ export async function runReplayScriptFile(params: { } } -async function invokeReplayAction(params: { - req: DaemonRequest; - sessionName: string; - action: SessionAction; - scope: ReplayVarScope; - filePath: string; - line: number; - step: number; - tracePath?: string; - invoke: (req: DaemonRequest) => Promise; -}): Promise { - const { req, sessionName, action, scope, filePath, line, step, tracePath, invoke } = params; - const resolved = resolveReplayAction(action, scope, { file: filePath, line }); - const startedAt = Date.now(); - appendReplayTraceEvent(tracePath, { - type: 'replay_action_start', - ts: new Date(startedAt).toISOString(), - replayPath: filePath, - line, - step, - command: resolved.command, - positionals: resolved.positionals ?? [], - }); - const response = await invoke({ - token: req.token, - session: sessionName, - command: resolved.command, - positionals: resolved.positionals ?? [], - flags: buildReplayActionFlags(req.flags, resolved.flags), - runtime: resolved.runtime, - meta: req.meta, - }); - const finishedAt = Date.now(); - appendReplayTraceEvent(tracePath, { - type: 'replay_action_stop', - ts: new Date(finishedAt).toISOString(), - replayPath: filePath, - line, - step, - command: resolved.command, - ok: response.ok, - durationMs: finishedAt - startedAt, - errorCode: response.ok ? undefined : response.error.code, - }); - return response; -} - -function appendReplayTraceEvent( - tracePath: string | undefined, - event: Record, -): void { - if (!tracePath) return; - fs.appendFileSync(tracePath, `${JSON.stringify(event)}\n`); -} - // fallow-ignore-next-line complexity function buildReplayBuiltinVars(params: { req: DaemonRequest; @@ -313,29 +256,21 @@ function isReplayArtifactPath(candidate: string): boolean { } } -export function buildReplayActionFlags( - parentFlags: CommandFlags | undefined, - actionFlags: SessionAction['flags'] | undefined, -): CommandFlags { - return mergeParentFlags(parentFlags, { ...(actionFlags ?? {}) }); -} - // fallow-ignore-next-line complexity function actionsContainInterpolation(actions: SessionAction[]): boolean { for (const action of actions) { for (const positional of action.positionals ?? []) { if (typeof positional === 'string' && positional.includes('${')) return true; } - if (action.flags) { - for (const value of Object.values(action.flags)) { - if (typeof value === 'string' && value.includes('${')) return true; - } - } - if (action.runtime) { - for (const value of Object.values(action.runtime)) { - if (typeof value === 'string' && value.includes('${')) return true; - } - } + if (containsInterpolation(action.flags)) return true; + if (containsInterpolation(action.runtime)) return true; } return false; } + +function containsInterpolation(value: unknown): boolean { + if (typeof value === 'string') return value.includes('${'); + if (Array.isArray(value)) return value.some(containsInterpolation); + if (value && typeof value === 'object') return Object.values(value).some(containsInterpolation); + return false; +} diff --git a/src/daemon/handlers/session.ts b/src/daemon/handlers/session.ts index 571c09f0b..08c0c05c3 100644 --- a/src/daemon/handlers/session.ts +++ b/src/daemon/handlers/session.ts @@ -203,13 +203,15 @@ export async function handleSessionCommands(params: { if (req.command === PUBLIC_COMMANDS.keyboard) { const session = sessionStore.get(sessionName); const keyboardAction = req.positionals?.[0]?.trim().toLowerCase(); - if (!session && keyboardAction === 'dismiss') { + const needsForegroundIosApp = + keyboardAction === 'dismiss' || keyboardAction === 'enter' || keyboardAction === 'return'; + if (!session && needsForegroundIosApp) { const flags = req.flags ?? {}; const normalizedPlatform = normalizePlatformSelector(flags.platform); if (normalizedPlatform === 'ios') { return errorResponse( 'SESSION_NOT_FOUND', - 'iOS keyboard dismiss requires an active session so the target app stays foregrounded. Run open first.', + 'iOS keyboard action requires an active session so the target app stays foregrounded. Run open first.', ); } } diff --git a/src/daemon/types.ts b/src/daemon/types.ts index d5f3720bd..aa5df7f59 100644 --- a/src/daemon/types.ts +++ b/src/daemon/types.ts @@ -228,6 +228,7 @@ export type SessionAction = { snapshotDepth?: number; snapshotScope?: string; snapshotRaw?: boolean; + launchArgs?: string[]; saveScript?: boolean | string; noRecord?: boolean; }; diff --git a/src/platforms/android/input-actions.ts b/src/platforms/android/input-actions.ts index 45c750521..4a43fd173 100644 --- a/src/platforms/android/input-actions.ts +++ b/src/platforms/android/input-actions.ts @@ -47,6 +47,10 @@ export async function homeAndroid(device: DeviceInfo): Promise { await runAndroidAdb(device, ['shell', 'input', 'keyevent', '3']); } +export async function pressAndroidEnter(device: DeviceInfo): Promise { + await runAndroidAdb(device, ['shell', 'input', 'keyevent', 'ENTER']); +} + export async function rotateAndroid( device: DeviceInfo, orientation: DeviceRotation, diff --git a/src/platforms/ios/__tests__/runner-client.test.ts b/src/platforms/ios/__tests__/runner-client.test.ts index cdcfeb02b..00ba17ad6 100644 --- a/src/platforms/ios/__tests__/runner-client.test.ts +++ b/src/platforms/ios/__tests__/runner-client.test.ts @@ -139,6 +139,7 @@ const runnerProtocolCommandFixtures: Record { const launchConsole = options?.launchConsole?.trim(); if (launchConsole && (device.platform !== 'ios' || device.kind !== 'simulator')) { @@ -185,7 +185,10 @@ export async function openIosApp( const bundleId = options?.appBundleId ?? (await resolveIosApp(device, app)); if (device.kind === 'simulator') { - await launchIosSimulatorApp(device, bundleId, launchConsole ? { launchConsole } : undefined); + await launchIosSimulatorApp(device, bundleId, { + ...(launchConsole ? { launchConsole } : {}), + ...(options?.launchArgs ? { launchArgs: options.launchArgs } : {}), + }); return; } @@ -235,6 +238,53 @@ export async function closeIosApp(device: DeviceInfo, app: string): Promise { + if (device.platform !== 'ios' || device.kind !== 'simulator') { + throw new AppError( + 'UNSUPPORTED_OPERATION', + 'Clearing app state is currently supported only on iOS simulators.', + ); + } + + const bundleId = await resolveIosApp(device, app); + await ensureBootedSimulator(device); + await closeIosApp(device, bundleId); + + const result = await runSimctl(device, ['get_app_container', device.id, bundleId, 'data'], { + allowFailure: true, + }); + if (result.exitCode !== 0) { + throw new AppError('COMMAND_FAILED', `simctl get_app_container failed for ${bundleId}`, { + stdout: result.stdout, + stderr: result.stderr, + exitCode: result.exitCode, + }); + } + + const containerPath = result.stdout.trim(); + if (!containerPath) { + throw new AppError( + 'COMMAND_FAILED', + `simctl get_app_container returned an empty data container path for ${bundleId}`, + ); + } + + const entries = await fs.readdir(containerPath); + await Promise.all( + entries.map((entry) => + fs.rm(path.join(containerPath, entry), { + recursive: true, + force: true, + }), + ), + ); + + return { bundleId, containerPath }; +} + export async function uninstallIosApp( device: DeviceInfo, app: string, @@ -884,7 +934,7 @@ function isIosBiometricCapabilityMissing(stdout: string, stderr: string): boolea async function launchIosSimulatorApp( device: DeviceInfo, bundleId: string, - options?: { launchConsole?: string }, + options?: { launchConsole?: string; launchArgs?: string[] }, ): Promise { await ensureBootedSimulator(device); @@ -947,11 +997,12 @@ async function launchIosSimulatorApp( function buildIosSimulatorLaunchArgs( deviceId: string, bundleId: string, - options?: { launchConsole?: string }, + options?: { launchConsole?: string; launchArgs?: string[] }, ): string[] { const args = ['launch']; if (options?.launchConsole) args.push('--console-pty'); args.push(deviceId, bundleId); + if (options?.launchArgs) args.push(...options.launchArgs); return args; } diff --git a/src/platforms/ios/interactions.ts b/src/platforms/ios/interactions.ts index 50f91f0df..4ec397caf 100644 --- a/src/platforms/ios/interactions.ts +++ b/src/platforms/ios/interactions.ts @@ -39,6 +39,7 @@ type IosRunnerOverrides = Pick< | 'longPress' | 'focus' | 'type' + | 'fillElementSelector' | 'fill' | 'scroll' | 'pinch' @@ -81,6 +82,7 @@ export function iosRunnerOverrides( command: 'tap', selectorKey: selector.key, selectorValue: selector.value, + allowNonHittableCoordinateFallback: selector.allowNonHittableCoordinateFallback, appBundleId: ctx.appBundleId, }, runnerOpts, @@ -159,7 +161,23 @@ export function iosRunnerOverrides( command: 'type', text, delayMs, - textEntryMode: 'append', + textEntryMode: text === '\n' ? undefined : 'append', + appBundleId: ctx.appBundleId, + }, + runnerOpts, + ); + }, + fillElementSelector: async (selector, text, delayMs) => { + return await runIosRunnerCommand( + device, + { + command: 'type', + selectorKey: selector.key, + selectorValue: selector.value, + allowNonHittableCoordinateFallback: selector.allowNonHittableCoordinateFallback, + text, + delayMs, + textEntryMode: 'replace', appBundleId: ctx.appBundleId, }, runnerOpts, diff --git a/src/platforms/ios/runner-contract.ts b/src/platforms/ios/runner-contract.ts index 598ed94f2..c7958a099 100644 --- a/src/platforms/ios/runner-contract.ts +++ b/src/platforms/ios/runner-contract.ts @@ -34,6 +34,7 @@ export type RunnerCommand = { | 'transformGesture' | 'appSwitcher' | 'keyboardDismiss' + | 'keyboardReturn' | 'alert' | 'pinch' | 'recordStart' @@ -44,6 +45,7 @@ export type RunnerCommand = { text?: string; selectorKey?: 'id' | 'label' | 'text' | 'value'; selectorValue?: string; + allowNonHittableCoordinateFallback?: boolean; delayMs?: number; textEntryMode?: 'append' | 'replace'; action?: 'get' | 'accept' | 'dismiss'; diff --git a/src/replay/vars.ts b/src/replay/vars.ts index dc3fb7d54..969bca5d0 100644 --- a/src/replay/vars.ts +++ b/src/replay/vars.ts @@ -2,7 +2,7 @@ import { AppError } from '../utils/errors.ts'; import type { SessionAction } from '../daemon/types.ts'; export type ReplayVarScope = { - readonly values: Readonly>; + readonly values: Record; }; export type ReplayVarSources = { @@ -13,7 +13,7 @@ export type ReplayVarSources = { }; export const REPLAY_VAR_KEY_RE = /^[A-Z_][A-Z0-9_]*$/; -const INTERPOLATION_RE = /(\\\$\{)|\$\{([A-Za-z_][A-Za-z0-9_]*)(?::-((?:[^}\\]|\\.)*))?\}/g; +const INTERPOLATION_RE = /(\\\$\{)|\$\{([A-Za-z_][A-Za-z0-9_.]*)(?::-((?:[^}\\]|\\.)*))?\}/g; const SHELL_PREFIX = 'AD_VAR_'; const RESERVED_NAMESPACE_PREFIX = 'AD_'; @@ -53,6 +53,13 @@ export function buildReplayVarScope(sources: ReplayVarSources): ReplayVarScope { return { values: merged }; } +export function mergeReplayVarScopeValues( + scope: ReplayVarScope, + values: Record, +): void { + Object.assign(scope.values, values); +} + export function collectReplayShellEnv(processEnv: NodeJS.ProcessEnv): Record { const result: Record = {}; for (const [rawKey, value] of Object.entries(processEnv)) { @@ -156,11 +163,20 @@ function resolveStringProps( loc: { file: string; line: number }, ): T | undefined { if (!obj) return obj; - const next: Record = { ...(obj as Record) }; - for (const [key, value] of Object.entries(next)) { - if (typeof value === 'string') { - next[key] = resolveReplayString(value, scope, loc); - } + return resolveStringValue(obj, scope, loc) as T; +} + +function resolveStringValue( + value: unknown, + scope: ReplayVarScope, + loc: { file: string; line: number }, +): unknown { + if (typeof value === 'string') return resolveReplayString(value, scope, loc); + if (Array.isArray(value)) return value.map((entry) => resolveStringValue(entry, scope, loc)); + if (value && typeof value === 'object') { + return Object.fromEntries( + Object.entries(value).map(([key, entry]) => [key, resolveStringValue(entry, scope, loc)]), + ); } - return next as T; + return value; } diff --git a/src/utils/__tests__/args.test.ts b/src/utils/__tests__/args.test.ts index 742e5f8dd..b8a1e6cc2 100644 --- a/src/utils/__tests__/args.test.ts +++ b/src/utils/__tests__/args.test.ts @@ -120,13 +120,14 @@ test('parseArgs recognizes command-specific flag combinations', async () => { }, { label: 'replay maestro flow', - argv: ['replay', './flow.yaml', '--maestro', '--env', 'USER=Ada'], + argv: ['replay', './flow.yaml', '--maestro', '--env', 'USER=Ada', '--timeout', '240000'], strictFlags: true, assertParsed: (parsed) => { assert.equal(parsed.command, 'replay'); assert.deepEqual(parsed.positionals, ['./flow.yaml']); assert.equal(parsed.flags.replayMaestro, true); assert.deepEqual(parsed.flags.replayEnv, ['USER=Ada']); + assert.equal(parsed.flags.timeoutMs, 240000); }, }, ]; @@ -369,6 +370,10 @@ test('parseArgs accepts keyboard subcommands', () => { const dismiss = parseArgs(['keyboard', 'dismiss'], { strictFlags: true }); assert.equal(dismiss.command, 'keyboard'); assert.deepEqual(dismiss.positionals, ['dismiss']); + + const enter = parseArgs(['keyboard', 'enter'], { strictFlags: true }); + assert.equal(enter.command, 'keyboard'); + assert.deepEqual(enter.positionals, ['enter']); }); test('parseArgs accepts scroll pixel distance flag', () => { @@ -917,10 +922,10 @@ test('usageForCommand includes Maestro replay flag', () => { assert.match(help, /--maestro/); assert.match(help, /doubleTapOn/); assert.match(help, /pasteText/); - assert.match(help, /setPermissions/); - assert.match(help, /startRecording\/stopRecording/); assert.match(help, /runFlow file\/inline/); + assert.match(help, /ordered trusted runScript/); assert.match(help, /repeat\.times/); + assert.match(help, /stopApp/); assert.match(help, /Unsupported syntax fails loudly/); assert.match(help, /issues\/558/); }); @@ -1417,8 +1422,11 @@ test('clipboard command usage is documented', () => { test('keyboard command usage is documented', () => { const help = usageForCommand('keyboard'); if (help === null) throw new Error('Expected command help text'); - assert.match(help, /keyboard \[status\|get\|dismiss\]/); - assert.match(help, /Inspect Android keyboard visibility\/type or dismiss the device keyboard/); + assert.match(help, /keyboard \[status\|get\|dismiss\|enter\|return\]/); + assert.match( + help, + /Inspect Android keyboard visibility\/type or press\/dismiss the device keyboard/, + ); }); test('rotate command usage is documented', () => { diff --git a/src/utils/__tests__/video.test.ts b/src/utils/__tests__/video.test.ts index 4e56c6b9e..cc1dc6b1c 100644 --- a/src/utils/__tests__/video.test.ts +++ b/src/utils/__tests__/video.test.ts @@ -3,6 +3,8 @@ import assert from 'node:assert/strict'; import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; +import { AppError } from '../errors.ts'; +import { withCommandExecutorOverride } from '../exec.ts'; import { isPlayableVideo } from '../video.ts'; function makeAtom(type: string, payload = Buffer.alloc(0)): Buffer { @@ -23,7 +25,7 @@ test('isPlayableVideo falls back to MP4 container validation when swift is unava process.env.AGENT_DEVICE_SWIFT_CACHE_DIR = path.join(tmpDir, 'swift-cache'); try { - assert.equal(await isPlayableVideo(videoPath), true); + assert.equal(await withUnavailableSwift(() => isPlayableVideo(videoPath)), true); } finally { process.env.PATH = previousPath; restoreEnv('AGENT_DEVICE_SWIFT_CACHE_DIR', previousSwiftCacheDir); @@ -42,7 +44,7 @@ test('isPlayableVideo fallback rejects files without playable MP4 atoms', async process.env.AGENT_DEVICE_SWIFT_CACHE_DIR = path.join(tmpDir, 'swift-cache'); try { - assert.equal(await isPlayableVideo(videoPath), false); + assert.equal(await withUnavailableSwift(() => isPlayableVideo(videoPath)), false); } finally { process.env.PATH = previousPath; restoreEnv('AGENT_DEVICE_SWIFT_CACHE_DIR', previousSwiftCacheDir); @@ -57,3 +59,9 @@ function restoreEnv(name: string, value: string | undefined): void { } process.env[name] = value; } + +async function withUnavailableSwift(fn: () => Promise): Promise { + return await withCommandExecutorOverride(() => { + throw new AppError('TOOL_MISSING', 'swift unavailable for test'); + }, fn); +} diff --git a/src/utils/command-schema.ts b/src/utils/command-schema.ts index 28710d73c..d29fa4985 100644 --- a/src/utils/command-schema.ts +++ b/src/utils/command-schema.ts @@ -1261,7 +1261,7 @@ const FLAG_DEFINITIONS: readonly FlagDefinition[] = [ type: 'boolean', usageLabel: '--maestro', usageDescription: - 'Replay: treat input as a Maestro YAML compatibility flow. Supported subset: launchApp without state-reset side effects, runFlow file/inline with when.platform, onFlowStart/onFlowComplete, deterministic repeat.times, tapOn, doubleTapOn, longPressOn, inputText, pasteText, openLink, assertVisible, assertNotVisible, assertTrue literal true/false, extendedWaitUntil, scroll, absolute/percentage swipe, takeScreenshot, hideKeyboard, pressKey back/enter/home, back, waitForAnimationToEnd, stopApp/killApp, setAirplaneMode, setLocation, setOrientation, supported setPermissions targets, and startRecording/stopRecording. ' + + 'Replay: treat input as a Maestro YAML compatibility flow. Supported subset: launchApp with Apple-platform launch arguments and iOS simulator clearState, runFlow file/inline with when.platform/visible/notVisible, ordered trusted runScript file/env steps with http.post/json/output variables, onFlowStart/onFlowComplete, deterministic repeat.times, tapOn including optional, index, childOf, label, and absolute/percentage point taps, doubleTapOn, longPressOn, inputText, eraseText for focused fields, pasteText, openLink, assertVisible, assertNotVisible, extendedWaitUntil, scroll, scrollUntilVisible, absolute/percentage swipe plus swipe.label, takeScreenshot, hideKeyboard, pressKey back/enter/home, back, waitForAnimationToEnd, and stopApp. ' + 'Unsupported syntax fails loudly with a link to https://github.com/callstackincubator/agent-device/issues/558', }, { @@ -1286,7 +1286,7 @@ const FLAG_DEFINITIONS: readonly FlagDefinition[] = [ type: 'int', min: 1, usageLabel: '--timeout ', - usageDescription: 'Test: maximum wall-clock time per script attempt', + usageDescription: 'Replay/Test: maximum wall-clock time per script attempt', }, { key: 'retries', @@ -1597,9 +1597,10 @@ const COMMAND_SCHEMAS: Record = { allowedFlags: [], }, keyboard: { - usageOverride: 'keyboard [status|get|dismiss]', - helpDescription: 'Inspect Android keyboard visibility/type or dismiss the device keyboard', - summary: 'Inspect or dismiss the device keyboard', + usageOverride: 'keyboard [status|get|dismiss|enter|return]', + helpDescription: + 'Inspect Android keyboard visibility/type or press/dismiss the device keyboard', + summary: 'Inspect, press, or dismiss the device keyboard', positionalArgs: ['action?'], allowedFlags: [], }, @@ -1674,7 +1675,7 @@ const COMMAND_SCHEMAS: Record = { replay: { helpDescription: 'Replay a recorded session', positionalArgs: ['path'], - allowedFlags: ['replayUpdate', 'replayMaestro', 'replayEnv'], + allowedFlags: ['replayUpdate', 'replayMaestro', 'replayEnv', 'timeoutMs'], skipCapabilityCheck: true, }, test: { diff --git a/website/docs/docs/replay-e2e.md b/website/docs/docs/replay-e2e.md index 675266000..a42384dcd 100644 --- a/website/docs/docs/replay-e2e.md +++ b/website/docs/docs/replay-e2e.md @@ -61,11 +61,11 @@ Maestro compatibility translates supported YAML commands into Agent Device repla - Supported and unsupported capabilities: https://github.com/callstackincubator/agent-device/issues/558 - New focused compatibility request: https://github.com/callstackincubator/agent-device/issues/new -Currently supported areas include app launch without state-reset side effects, file and inline `runFlow` with `when.platform`, `onFlowStart` / `onFlowComplete`, deterministic `repeat.times`, `tapOn`, `doubleTapOn`, `longPressOn`, `inputText`, `pasteText`, `openLink`, visibility assertions, literal `assertTrue`, `extendedWaitUntil`, `scroll`, absolute/percentage `swipe`, screenshots, keyboard dismiss, basic `pressKey`, `back`, animation waits, `stopApp` / `killApp`, airplane mode, mock location, orientation, supported permission targets, and screen recording. +Currently supported areas include app launch with Apple-platform launch arguments and iOS simulator `clearState`, file and inline `runFlow` with `when.platform`, `when.visible`, and `when.notVisible`, `onFlowStart` / `onFlowComplete`, deterministic `repeat.times`, `tapOn` including `optional`, `index`, `childOf`, `label`, and absolute/percentage point taps, `doubleTapOn`, `longPressOn`, `inputText`, focused-field `eraseText`, `pasteText`, `openLink`, visibility assertions, `extendedWaitUntil`, `scroll`, `scrollUntilVisible`, absolute/percentage `swipe`, `swipe.label`, screenshots, keyboard dismiss, basic `pressKey`, `back`, animation waits, and `stopApp`. `runScript` is supported only as an ordered Maestro compatibility step for trusted file/env scripts that use `http.post`, `json`, and `output` variables; it can make network requests, and is not a native `.ad` command or security sandbox. Script execution uses Node `vm` only for compatibility isolation, not for security; the script timeout bounds synchronous execution, while `http.post` requests are bounded by the helper process timeout. Output keys cannot contain `.` because exported variables are addressed as `output.`. Maestro `env` values use the same replay precedence as `.ad` files: flow `env` is the default, shell `AD_VAR_*` values override it, and CLI `-e KEY=VALUE` wins over both. -Runtime-dependent Maestro features such as `scrollUntilVisible`, `repeat.while`, `runFlow.when.visible`, `runScript`, `evalScript`, text clearing, and app state reset are tracked separately because they require neutral Agent Device runtime or device capabilities before they can be mapped safely. +Unsupported Maestro features such as `repeat.while`, `runFlow.when.true`, full expression predicates, `evalScript`, device utility commands, Android app launch arguments, and Android app state reset are tracked separately because they require neutral Agent Device runtime or device capabilities before they can be mapped safely. ## Run a lightweight `.ad` suite