diff --git a/packages/contracts/src/client-gesture.ts b/packages/contracts/src/client-gesture.ts index 998f547143..823f318e4f 100644 --- a/packages/contracts/src/client-gesture.ts +++ b/packages/contracts/src/client-gesture.ts @@ -81,6 +81,14 @@ export type PanOptions = DeviceCommandBaseOptions & { durationMs?: number; }; +export type DragOptions = DeviceCommandBaseOptions & { + source: string; + destination: string; + sourceHoldMs?: number; + moveMs?: number; + destinationHoldMs?: number; +}; + export type FlingOptions = DeviceCommandBaseOptions & { direction: ScrollDirection; x: number; diff --git a/packages/contracts/src/gesture-input.ts b/packages/contracts/src/gesture-input.ts index ad9fb62244..ff22368866 100644 --- a/packages/contracts/src/gesture-input.ts +++ b/packages/contracts/src/gesture-input.ts @@ -14,6 +14,16 @@ import { } from './gesture-plan-types.ts'; export const GESTURE_KINDS = ['pan', 'fling', 'swipe', 'pinch', 'rotate', 'transform'] as const; +export const GESTURE_INPUT_KINDS = [...GESTURE_KINDS, 'drag'] as const; + +export type DragGesturePayload = { + kind: 'drag'; + source: string; + destination: string; + sourceHoldMs?: number; + moveMs?: number; + destinationHoldMs?: number; +}; export type PanGesturePayload = { kind: 'pan'; @@ -62,11 +72,12 @@ export type GesturePayload = | SwipeGesturePayload | PinchGesturePayload | RotateGesturePayload - | TransformGesturePayload; + | TransformGesturePayload + | DragGesturePayload; export function readGesturePayload(input: unknown): GesturePayload { const record = readRecord(input); - const kind = readEnum(record, 'kind', GESTURE_KINDS); + const kind = readEnum(record, 'kind', GESTURE_INPUT_KINDS); if (kind === 'pan') { return { kind, @@ -78,6 +89,19 @@ export function readGesturePayload(input: unknown): GesturePayload { durationMs: readOptionalGestureDuration(record), }; } + if (kind === 'drag') { + return { + kind, + source: readNonEmptyString(record, 'source'), + destination: readNonEmptyString(record, 'destination'), + sourceHoldMs: readOptionalInteger(record, 'sourceHoldMs', { min: 1, max: 10_000 }), + moveMs: readOptionalInteger(record, 'moveMs', { min: 16, max: 10_000 }), + destinationHoldMs: readOptionalInteger(record, 'destinationHoldMs', { + min: 0, + max: 10_000, + }), + }; + } if (record.pointerCount !== undefined) { throw new AppError('INVALID_ARGS', 'pointerCount is supported only for gesture pan'); } @@ -173,6 +197,14 @@ function readNumber(record: Record, key: string): number { return value; } +function readNonEmptyString(record: Record, key: string): string { + const value = record[key]; + if (typeof value !== 'string' || value.trim().length === 0) { + throw new AppError('INVALID_ARGS', `Expected ${key} to be a non-empty string.`); + } + return value; +} + function readEnum( record: Record, key: string, diff --git a/packages/contracts/src/gesture-normalization.test.ts b/packages/contracts/src/gesture-normalization.test.ts index 1ac2bfb136..8004f40693 100644 --- a/packages/contracts/src/gesture-normalization.test.ts +++ b/packages/contracts/src/gesture-normalization.test.ts @@ -58,6 +58,18 @@ test('gesture recording codec round-trips fling with distance', () => { assert.deepEqual(gesturePayloadFromPositionals(gesturePayloadToPositionals(payload)), payload); }); +test('gesture recording codec round-trips selector-authored hold drag timings', () => { + const payload = { + kind: 'drag' as const, + source: 'label="Blue card"', + destination: '@destination', + sourceHoldMs: 800, + moveMs: 700, + destinationHoldMs: 250, + }; + assert.deepEqual(gesturePayloadFromPositionals(gesturePayloadToPositionals(payload)), payload); +}); + test('a retired trailing positional reports its migration, not a bare usage line', () => { assert.throws(() => swipePayloadFromPositionals(['197', '650', '197', '300', '300']), { code: 'INVALID_ARGS', diff --git a/packages/contracts/src/gesture-normalization.ts b/packages/contracts/src/gesture-normalization.ts index 73e26f96cb..a88114556d 100644 --- a/packages/contracts/src/gesture-normalization.ts +++ b/packages/contracts/src/gesture-normalization.ts @@ -1,6 +1,10 @@ import type { Point } from '@agent-device/kernel/snapshot'; import { AppError } from '@agent-device/kernel/errors'; -import { readGesturePayload, type GESTURE_KINDS, type GesturePayload } from './gesture-input.ts'; +import { + readGesturePayload, + type GESTURE_INPUT_KINDS, + type GesturePayload, +} from './gesture-input.ts'; import type { GestureSemanticInput } from './gesture-plan-types.ts'; export type NormalizedPublicGesture = { @@ -16,7 +20,7 @@ export type SwipePayload = { }; /** Derived from the canonical kinds, so a new gesture kind cannot skip the arity table. */ -type GestureSyntaxKey = 'swipe' | `gesture ${(typeof GESTURE_KINDS)[number]}`; +type GestureSyntaxKey = 'swipe' | `gesture ${(typeof GESTURE_INPUT_KINDS)[number]}`; type PublicGestureSyntax = { /** Highest accepted positional count, flags excluded. */ @@ -81,6 +85,11 @@ const PUBLIC_GESTURE_SYNTAX: Record = { max: 7, usage: 'gesture transform accepts at most 7 arguments: x y dx dy scale degrees [durationMs]', }, + 'gesture drag': { + max: 5, + usage: + 'gesture drag accepts at most 5 arguments: source destination [sourceHoldMs] [moveMs] [destinationHoldMs]', + }, }; /** `swipe x1 y1 x2 y2 durationMs` translates to the equivalent timed pan. */ @@ -256,6 +265,17 @@ export function gesturePayloadFromPositionals( durationMs: optionalPositionNumber(args[6]), }); } + case 'drag': { + assertGestureArity('gesture drag', args); + return readGesturePayload({ + kind, + source: args[0], + destination: args[1], + sourceHoldMs: optionalPositionNumber(args[2]), + moveMs: optionalPositionNumber(args[3]), + destinationHoldMs: optionalPositionNumber(args[4]), + }); + } default: return readGesturePayload({ kind }); } @@ -294,6 +314,15 @@ export function gesturePayloadToPositionals(input: GesturePayload): string[] { input.degrees, input.durationMs, ]); + case 'drag': + return compact([ + input.kind, + input.source, + input.destination, + input.sourceHoldMs, + input.moveMs, + input.destinationHoldMs, + ]); } } @@ -355,6 +384,11 @@ export function normalizePublicGesture(input: GesturePayload): NormalizedPublicG durationMs: input.durationMs, }, }; + case 'drag': + throw new AppError( + 'INVALID_ARGS', + 'gesture drag targets must be resolved by the interaction runtime', + ); } } diff --git a/packages/contracts/src/gesture-plan-types.ts b/packages/contracts/src/gesture-plan-types.ts index 94bf7f7dd3..46cf9f0fd6 100644 --- a/packages/contracts/src/gesture-plan-types.ts +++ b/packages/contracts/src/gesture-plan-types.ts @@ -8,6 +8,15 @@ export const GESTURE_DURATION_MAX_MS = 10_000; export type GestureIntent = 'fling' | 'pan' | 'pinch' | 'rotate' | 'transform'; +export type DragGestureTargetInput = { + intent: 'drag'; + source: string; + destination: string; + sourceHoldMs?: number; + moveMs?: number; + destinationHoldMs?: number; +}; + /** Selects one-pointer release timing without changing semantic gesture intent. */ export type GestureExecutionProfile = 'endpoint-hold' | 'timed-pan'; @@ -39,6 +48,8 @@ export type GestureSemanticInput = durationMs?: number; }; +export type PublicGestureSemanticInput = GestureSemanticInput | DragGestureTargetInput; + export type PointerTrajectorySample = { offsetMs: number; point: Point }; export type PointerTrajectory = { diff --git a/packages/contracts/src/gesture-plan.test.ts b/packages/contracts/src/gesture-plan.test.ts new file mode 100644 index 0000000000..e621dc17a6 --- /dev/null +++ b/packages/contracts/src/gesture-plan.test.ts @@ -0,0 +1,47 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { buildHoldDragGesturePlan } from './gesture-plan.ts'; + +test('hold drag keeps one pointer down through source hold, movement, and destination hold', () => { + const plan = buildHoldDragGesturePlan( + { + from: { x: 20, y: 30 }, + to: { x: 120, y: 230 }, + sourceHoldMs: 800, + moveMs: 700, + destinationHoldMs: 250, + }, + { x: 0, y: 0, width: 400, height: 800 }, + 'ios', + ); + + assert.equal(plan.topology, 'single'); + assert.equal(plan.durationMs, 1_750); + assert.deepEqual(plan.pointers[0]?.samples[0], { offsetMs: 0, point: { x: 20, y: 30 } }); + assert.deepEqual(plan.pointers[0]?.samples[1], { offsetMs: 800, point: { x: 20, y: 30 } }); + assert.deepEqual(plan.pointers[0]?.samples.at(-2), { + offsetMs: 1_500, + point: { x: 120, y: 230 }, + }); + assert.deepEqual(plan.pointers[0]?.samples.at(-1), { + offsetMs: 1_750, + point: { x: 120, y: 230 }, + }); +}); + +test('hold drag rejects a combined duration above the backend ceiling', () => { + assert.throws( + () => + buildHoldDragGesturePlan( + { + from: { x: 20, y: 30 }, + to: { x: 120, y: 230 }, + sourceHoldMs: 5_000, + moveMs: 5_000, + destinationHoldMs: 1, + }, + { x: 0, y: 0, width: 400, height: 800 }, + ), + { code: 'INVALID_ARGS', message: 'gesture drag total duration must be at most 10000' }, + ); +}); diff --git a/packages/contracts/src/gesture-plan.ts b/packages/contracts/src/gesture-plan.ts index 2115b2f174..275a7d92ea 100644 --- a/packages/contracts/src/gesture-plan.ts +++ b/packages/contracts/src/gesture-plan.ts @@ -30,6 +30,9 @@ const GESTURE_VIEWPORT_INSET_PX = 1; const DEFAULT_PAN_DURATION_MS = 500; export const GESTURE_FLING_DURATION_MS = 100; const DEFAULT_MULTI_TOUCH_DURATION_MS = 300; +export const DEFAULT_DRAG_SOURCE_HOLD_MS = 800; +export const DEFAULT_DRAG_MOVE_MS = 500; +export const DEFAULT_DRAG_DESTINATION_HOLD_MS = 0; const MAX_ROTATION_DEGREES_PER_SAMPLE = 3; const MAX_ROTATION_DEFAULT_DURATION_MS = 2_400; @@ -129,6 +132,65 @@ export function singlePointerPlanEndpoints(plan: SinglePointerGesturePlan): { return { start, end }; } +/** + * PROTOTYPE: one uninterrupted contact for reorder-style hold, move, hold, release. + * Element targets are resolved by the runtime before this portable planning seam. + */ +export function buildHoldDragGesturePlan( + input: { + from: Point; + to: Point; + sourceHoldMs?: number; + moveMs?: number; + destinationHoldMs?: number; + }, + viewport: Rect, + platform?: PublicPlatform, +): SinglePointerGesturePlan { + const frame = normalizeViewport(viewport); + const profile = gesturePlatformProfile(platform); + const start = finitePoint(input.from, 'gesture drag source'); + const end = finitePoint(input.to, 'gesture drag destination'); + const sourceHoldMs = normalizePositiveDuration( + input.sourceHoldMs, + DEFAULT_DRAG_SOURCE_HOLD_MS, + 'gesture drag sourceHoldMs', + ); + const moveMs = normalizeDuration(input.moveMs, DEFAULT_DRAG_MOVE_MS, 'gesture drag moveMs'); + const destinationHoldMs = normalizeNonNegativeDuration( + input.destinationHoldMs, + DEFAULT_DRAG_DESTINATION_HOLD_MS, + 'gesture drag destinationHoldMs', + ); + const durationMs = sourceHoldMs + moveMs + destinationHoldMs; + if (durationMs > GESTURE_DURATION_MAX_MS) { + throw new AppError( + 'INVALID_ARGS', + `gesture drag total duration must be at most ${GESTURE_DURATION_MAX_MS}`, + ); + } + const samples = [ + { offsetMs: 0, point: start }, + { offsetMs: sourceHoldMs, point: start }, + ...sampleOffsets(moveMs, profile) + .slice(1) + .map((moveOffsetMs) => ({ + offsetMs: sourceHoldMs + moveOffsetMs, + point: interpolatePoint(start, end, moveOffsetMs / moveMs), + })), + ...(destinationHoldMs > 0 ? [{ offsetMs: durationMs, point: end }] : []), + ]; + assertSamplesInViewport(samples, frame, { intent: 'drag', pointerId: 0 }); + return { + topology: 'single', + intent: 'pan', + executionProfile: 'timed-pan', + durationMs, + viewport: frame, + pointers: [{ pointerId: 0, samples }], + }; +} + function buildFlingPlan( input: Extract, viewport: Rect, @@ -402,6 +464,28 @@ function normalizeDuration(value: number | undefined, fallback: number, field: s return durationMs; } +function normalizePositiveDuration(value: number | undefined, fallback: number, field: string) { + const durationMs = value ?? fallback; + if (!Number.isInteger(durationMs) || durationMs < 1 || durationMs > GESTURE_DURATION_MAX_MS) { + throw new AppError( + 'INVALID_ARGS', + `${field} must be an integer between 1 and ${GESTURE_DURATION_MAX_MS}`, + ); + } + return durationMs; +} + +function normalizeNonNegativeDuration(value: number | undefined, fallback: number, field: string) { + const durationMs = value ?? fallback; + if (!Number.isInteger(durationMs) || durationMs < 0 || durationMs > GESTURE_DURATION_MAX_MS) { + throw new AppError( + 'INVALID_ARGS', + `${field} must be an integer between 0 and ${GESTURE_DURATION_MAX_MS}`, + ); + } + return durationMs; +} + function defaultTransformDuration(rotationDegrees: number): number { const rotationDuration = Math.ceil(Math.abs(rotationDegrees) / MAX_ROTATION_DEGREES_PER_SAMPLE) * diff --git a/scripts/prototypes/continuous-hold-drag/README.md b/scripts/prototypes/continuous-hold-drag/README.md new file mode 100644 index 0000000000..764d9de0ec --- /dev/null +++ b/scripts/prototypes/continuous-hold-drag/README.md @@ -0,0 +1,119 @@ +# Continuous hold-drag prototype + +> THROWAWAY PROTOTYPE for +> [react-native-reorderable issue 21](https://github.com/thiagobrez/react-native-reorderable/issues/21). + +## Question + +Can agent-device synthesize one uninterrupted pointer lifecycle—source hold, +movement, optional destination hold, then release—while preserving +accessibility selectors in deterministic `.ad` recordings? + +The prototype adds this experimental syntax: + +```sh +pnpm ad gesture drag \ + 'label="Blue card, position 1"' \ + 'label="Yellow card, position 3"' \ + 800 700 250 \ + --session reorder --platform ios --device 'iPhone 17 Pro' +``` + +The timing arguments are `sourceHoldMs`, `moveMs`, and `destinationHoldMs`. +The runtime resolves both selectors immediately before dispatch, then lowers +them to the existing cross-platform timestamped pointer trajectory. Repeated +source and destination samples create the holds without releasing contact. + +## Evidence + +### Native SwiftUI reorder on iOS 27 + +The selector-authored gesture moved Blue from position 1 to position 3. The +post-drop accessibility tree exposed exactly five rows in this order: + +```text +Green card, position 1 +Yellow card, position 2 +Blue card, position 3 +Orange card, position 4 +Pink card, position 5 +``` + +The side-by-side probe independently exposed `native dropped → G · Y · B · O · P`. +The active-drag frames in [native-ios-contact-sheet.png](native-ios-contact-sheet.png) +show SwiftUI's translucent source/preview and live destination placeholder +before release. The primary recording is [native-ios.mp4](native-ios.mp4). + +### Reanimated + Gesture Handler fallback on iOS 27 + +The same command surface moved fallback Blue below Green. The probe exposed +one begin, one threshold crossing, and one drop: + +```text +fallback began blue +fallback center 95pt crossed into index 1 +fallback dropped blue → G · B · Y · O · P +``` + +The active-drag frames in +[fallback-ios-contact-sheet.png](fallback-ios-contact-sheet.png) show Green +displaced and the outlined Blue row held at the pending insertion point before +release. The primary recording is [fallback-ios.mp4](fallback-ios.mp4). + +### Reanimated + Gesture Handler fallback on Android 17 + +The same accessibility-label-authored gesture (`Fallback Blue card` to +`Fallback Yellow card`) ran on the Pixel 10 Pro emulator against the fallback +probe's 220 ms long-press activation threshold. Blue moved below Yellow, and +the Android accessibility tree exposed the complete lifecycle and resulting +order: + +```text +fallback began blue +fallback center 88pt crossed into index 1 +fallback center 159pt crossed into index 2 +fallback dropped blue → G · Y · B · O · P +``` + +The active-drag frames in +[android-fallback-contact-sheet.png](android-fallback-contact-sheet.png) show +the Blue row moving continuously while Green and Yellow occupy their pending +positions. The primary recording is +[android-fallback.mp4](android-fallback.mp4). + +### Deterministic recording and replay + +[native-ios.ad](native-ios.ad) retains both accessibility selectors and all +three timing phases. From a clean source daemon, public replay completed its +three steps in 9.8 seconds and the destination guard verified +`Blue card, position 3`. A post-replay snapshot independently confirmed the +five-row order above. + +[android-fallback.ad](android-fallback.ad) records the Android device context, +source readiness guard, selectors, timings, and destination-order guard. Its +four steps replayed headlessly in 11.2 seconds from a clean daemon. + +The first replay attempt failed before gesture dispatch because the authoring +daemon retained the XCTest runner lease. Stopping that daemon, as required by +`agent-device help validate`, made the identical script pass. CI must preserve +that prepare/authoring-daemon handoff rule. + +## Status + +- PASS: one uninterrupted hold → move → destination hold → release on iOS. +- PASS: element-to-element targeting through accessibility labels, without + test-only IDs. +- PASS: native SwiftUI reorder and fallback reorder on iOS 27. +- PASS: fallback long-press activation, continuous drag, boundary crossings, + drop callback, and final order on an Android 17 Pixel 10 Pro emulator. +- PASS: selector-preserving `.ad` publication and clean-daemon headless replay. +- PASS: observable active destination feedback, callback trace, and resulting + order on both iOS engines. +- PASS: the same timestamped pointer plan dispatches through the Android + MotionEvent backend without releasing contact before the final sample. + +The iOS and Android results establish that the architecture is viable for the +issue's required native and fallback harness lanes. This Android probe is +deliberately smaller than the planned production fallback engine; it proves +gesture synthesis and observable reorder behavior, not the full portable +contract. diff --git a/scripts/prototypes/continuous-hold-drag/android-fallback-contact-sheet.png b/scripts/prototypes/continuous-hold-drag/android-fallback-contact-sheet.png new file mode 100644 index 0000000000..37836515ae Binary files /dev/null and b/scripts/prototypes/continuous-hold-drag/android-fallback-contact-sheet.png differ diff --git a/scripts/prototypes/continuous-hold-drag/android-fallback.ad b/scripts/prototypes/continuous-hold-drag/android-fallback.ad new file mode 100644 index 0000000000..4b0bfdfac3 --- /dev/null +++ b/scripts/prototypes/continuous-hold-drag/android-fallback.ad @@ -0,0 +1,6 @@ +context platform=android device="Pixel 10 Pro" kind=emulator theme=unknown +open "reorderable.example" --relaunch --platform android --metro-host 127.0.0.1 --metro-port 8082 +wait "label=\"Fallback Blue card\"" +gesture "drag" "label=\"Fallback Blue card\"" "label=\"Fallback Yellow card\"" 800 900 1500 +# agent-device:target-v1 {"id":"fallback-order","role":"textview","label":"G · Y · B · O · P","ancestry":[{"role":"viewgroup"},{"role":"framelayout"},{"role":"framelayout"},{"role":"linearlayout"},{"role":"framelayout"},{"role":"linearlayout"},{"role":"framelayout"}],"sibling":7,"viewportOrder":0,"rect":{"x":652,"y":554,"width":592,"height":37},"verification":"verified"} +wait "label=\"G · Y · B · O · P\"" diff --git a/scripts/prototypes/continuous-hold-drag/android-fallback.mp4 b/scripts/prototypes/continuous-hold-drag/android-fallback.mp4 new file mode 100644 index 0000000000..37e694ce1d Binary files /dev/null and b/scripts/prototypes/continuous-hold-drag/android-fallback.mp4 differ diff --git a/scripts/prototypes/continuous-hold-drag/fallback-ios-contact-sheet.png b/scripts/prototypes/continuous-hold-drag/fallback-ios-contact-sheet.png new file mode 100644 index 0000000000..dafb0aff56 Binary files /dev/null and b/scripts/prototypes/continuous-hold-drag/fallback-ios-contact-sheet.png differ diff --git a/scripts/prototypes/continuous-hold-drag/fallback-ios.mp4 b/scripts/prototypes/continuous-hold-drag/fallback-ios.mp4 new file mode 100644 index 0000000000..4ee2d62cc9 Binary files /dev/null and b/scripts/prototypes/continuous-hold-drag/fallback-ios.mp4 differ diff --git a/scripts/prototypes/continuous-hold-drag/native-ios-contact-sheet.png b/scripts/prototypes/continuous-hold-drag/native-ios-contact-sheet.png new file mode 100644 index 0000000000..38f0cc6a66 Binary files /dev/null and b/scripts/prototypes/continuous-hold-drag/native-ios-contact-sheet.png differ diff --git a/scripts/prototypes/continuous-hold-drag/native-ios.ad b/scripts/prototypes/continuous-hold-drag/native-ios.ad new file mode 100644 index 0000000000..538e032354 --- /dev/null +++ b/scripts/prototypes/continuous-hold-drag/native-ios.ad @@ -0,0 +1,5 @@ +context platform=ios device="iPhone 17 Pro" kind=simulator theme=unknown +open "reorderable.example" --relaunch +gesture "drag" "label=\"Blue card, position 1\"" "label=\"Yellow card, position 3\"" 800 700 250 +# agent-device:target-v1 {"role":"other","label":"Blue card, position 3","ancestry":[{"role":"other","label":"Green card, position 1"},{"role":"other","label":"Single collection cards"},{"role":"other","label":"Cards"},{"role":"application","label":"ReorderableExample"}],"sibling":2,"viewportOrder":0,"rect":{"x":16,"y":297.6666717529297,"width":370,"height":54},"verification":"verified"} +wait "label=\"Blue card, position 3\"" 10000 diff --git a/scripts/prototypes/continuous-hold-drag/native-ios.mp4 b/scripts/prototypes/continuous-hold-drag/native-ios.mp4 new file mode 100644 index 0000000000..62fec6a922 Binary files /dev/null and b/scripts/prototypes/continuous-hold-drag/native-ios.mp4 differ diff --git a/src/agent-device-client.ts b/src/agent-device-client.ts index f95eeec288..f936a00383 100644 --- a/src/agent-device-client.ts +++ b/src/agent-device-client.ts @@ -15,6 +15,7 @@ import type { CaptureScreenshotResult, CaptureSnapshotOptions, CaptureSnapshotResult, + DragOptions, FlingOptions, InternalRequestOptions, Lease, @@ -400,6 +401,7 @@ export function createAgentDeviceClient( longPress: async (options) => await executeCommand('longpress', options), swipe: async (options) => await executeCommand('swipe', options), pan: async (options) => await executeCommand('gesture', panGestureInput(options)), + drag: async (options) => await executeCommand('gesture', dragGestureInput(options)), fling: async (options) => await executeCommand('gesture', flingGestureInput(options)), swipeGesture: async (options) => await executeCommand('gesture', swipePresetGestureInput(options)), @@ -452,6 +454,10 @@ function panGestureInput(options: PanOptions): InternalRequestOptions & Record { + return { ...options, kind: 'drag' }; +} + function flingGestureInput( options: FlingOptions, ): InternalRequestOptions & Record { diff --git a/src/cli/parser/__tests__/cli-help-topics.test.ts b/src/cli/parser/__tests__/cli-help-topics.test.ts index 576ddf97d1..5ac5f13c7e 100644 --- a/src/cli/parser/__tests__/cli-help-topics.test.ts +++ b/src/cli/parser/__tests__/cli-help-topics.test.ts @@ -19,7 +19,7 @@ test('usage includes concise top-level commands', async () => { assert.match(usageText, /clipboard read \| clipboard write /); assert.match(usageText, /keyboard \[action\]/); assert.match(usageText, /trigger-app-event\s{2,}Invoke app-defined automation\/test events/); - assert.match(usageText, /gesture \.\.\./); + assert.match(usageText, /gesture \.\.\./); assert.doesNotMatch( usageText, /install-from-source \| install-from-source --github-actions-artifact/, diff --git a/src/client/client-types.ts b/src/client/client-types.ts index cbdd92a048..a8507687de 100644 --- a/src/client/client-types.ts +++ b/src/client/client-types.ts @@ -85,6 +85,7 @@ import type { DeviceCommandBaseOptions, DeviceShutdownOptions, DoctorCommandOptions, + DragOptions, EventsOptions, FillOptions, FindOptions, @@ -272,6 +273,7 @@ export type AgentDeviceClient = { longPress: (options: LongPressOptions) => Promise>; swipe: (options: SwipeOptions) => Promise; pan: (options: PanOptions) => Promise; + drag: (options: DragOptions) => Promise; fling: (options: FlingOptions) => Promise; swipeGesture: (options: SwipeGestureOptions) => Promise; focus: (options: FocusOptions) => Promise; diff --git a/src/commands/interaction/index.ts b/src/commands/interaction/index.ts index a16538df4a..7d3aa7d8ce 100644 --- a/src/commands/interaction/index.ts +++ b/src/commands/interaction/index.ts @@ -1,5 +1,6 @@ import type { ClickOptions, + DragOptions, FillOptions, FindOptions, FlingOptions, @@ -42,6 +43,7 @@ import { interactionCliReaders, interactionDaemonWriters } from './interactions. import { interactionCommandMetadata, type ClickInput, + type DragInput, type FillInput, type FlingInput, type GetInput, @@ -116,11 +118,11 @@ const interactionCliSchemas = { allowedFlags: ['count', 'pauseMs', 'pattern'], }, gesture: { - usageOverride: 'gesture ...', - listUsageOverride: 'gesture ...', + usageOverride: 'gesture ...', + listUsageOverride: 'gesture ...', helpDescription: - 'Run touch gestures: pan [durationMs], fling [distance], swipe , pinch [x] [y], rotate [x] [y], or transform [durationMs]. For command plans, output only command lines. Android transform verification should use all app-observable effects, for example wait text "pan changed yes", wait text "pinch changed yes", and wait text "rotate changed yes", not exact transform values.', - summary: 'Run pan, fling, swipe, pinch, rotate, or transform gestures', + 'Run touch gestures: pan [durationMs], fling [distance], swipe , pinch [x] [y], rotate [x] [y], transform [durationMs], or drag [sourceHoldMs] [moveMs] [destinationHoldMs]. For command plans, output only command lines. Android transform verification should use all app-observable effects, for example wait text "pan changed yes", wait text "pinch changed yes", and wait text "rotate changed yes", not exact transform values.', + summary: 'Run pan, fling, swipe, pinch, rotate, transform, or hold-drag gestures', positionalArgs: ['pan|fling|swipe|pinch|rotate|transform', 'args?'], allowsExtraPositionals: true, allowedFlags: ['pointerCount'], @@ -222,6 +224,8 @@ const gestureCommandDefinition = defineExecutableCommand( return await client.interactions.rotateGesture(toRotateOptions(input)); case 'transform': return await client.interactions.transformGesture(toTransformOptions(input)); + case 'drag': + return await client.interactions.drag(toDragOptions(input)); } }, ); @@ -455,6 +459,17 @@ function toPanOptions(input: PanInput): PanOptions { }; } +function toDragOptions(input: DragInput): DragOptions { + return { + ...commonToClientOptions(input), + source: input.source, + destination: input.destination, + sourceHoldMs: input.sourceHoldMs, + moveMs: input.moveMs, + destinationHoldMs: input.destinationHoldMs, + }; +} + function toFlingOptions(input: FlingInput): FlingOptions { return { ...commonToClientOptions(input), diff --git a/src/commands/interaction/metadata.ts b/src/commands/interaction/metadata.ts index c37cf16756..cc1685edd5 100644 --- a/src/commands/interaction/metadata.ts +++ b/src/commands/interaction/metadata.ts @@ -1,6 +1,6 @@ import { CLICK_BUTTONS, - GESTURE_KINDS, + GESTURE_INPUT_KINDS, readGesturePayload, SCROLL_DIRECTIONS, SCROLL_DURATION_MAX_MS, @@ -9,6 +9,7 @@ import { SWIPE_PRESETS, SWIPE_REPETITION_MAX, type FlingGesturePayload, + type DragGesturePayload, type PanGesturePayload, type PinchGesturePayload, type RotateGesturePayload, @@ -210,7 +211,7 @@ const findFields = { }; const gestureFields = { - kind: requiredField(enumField(GESTURE_KINDS, 'Gesture variant.')), + kind: requiredField(enumField(GESTURE_INPUT_KINDS, 'Gesture variant.')), direction: enumField(SCROLL_DIRECTIONS, 'Fling direction.'), preset: enumField(SWIPE_PRESETS, 'Swipe preset.'), origin: pointField('Gesture origin point.'), @@ -220,6 +221,14 @@ const gestureFields = { degrees: numberField('Rotation in degrees.'), durationMs: integerField('Pan/transform duration.', { min: 16, max: 10_000 }), pointerCount: integerField('Pan touch pointer count (1 or 2).', { min: 1, max: 2 }), + source: stringField('Drag source @ref or selector.'), + destination: stringField('Drag destination @ref or selector.'), + sourceHoldMs: integerField('Drag activation hold duration.', { min: 1, max: 10_000 }), + moveMs: integerField('Drag movement duration.', { min: 16, max: 10_000 }), + destinationHoldMs: integerField('Hold before releasing at the destination.', { + min: 0, + max: 10_000, + }), }; export type ClickInput = InferCommandInput; @@ -234,6 +243,7 @@ export type SwipeGestureInput = CommonCommandInput & SwipeGesturePayload; export type PinchInput = CommonCommandInput & PinchGesturePayload; export type RotateInput = CommonCommandInput & RotateGesturePayload; export type TransformInput = CommonCommandInput & TransformGesturePayload; +export type DragInput = CommonCommandInput & DragGesturePayload; export type GestureInput = | PanInput @@ -241,7 +251,8 @@ export type GestureInput = | SwipeGestureInput | PinchInput | RotateInput - | TransformInput; + | TransformInput + | DragInput; export const interactionCommandMetadata = [ defineCommandMetadata({ diff --git a/src/commands/interaction/runtime/gesture-command.ts b/src/commands/interaction/runtime/gesture-command.ts index fdc49c459d..c3efa781bc 100644 --- a/src/commands/interaction/runtime/gesture-command.ts +++ b/src/commands/interaction/runtime/gesture-command.ts @@ -1,6 +1,11 @@ import type { AgentDeviceRuntime, CommandContext } from '../../../runtime-contract.ts'; -import type { GestureIntent, GestureSemanticInput } from '@agent-device/contracts/interaction'; -import { buildGesturePlan } from '@agent-device/contracts/interaction'; +import type { + DragGestureTargetInput, + GestureIntent, + GestureSemanticInput, + PublicGestureSemanticInput, +} from '@agent-device/contracts/interaction'; +import { buildGesturePlan, buildHoldDragGesturePlan } from '@agent-device/contracts/interaction'; import type { Point, Rect } from '@agent-device/kernel/snapshot'; import { AppError } from '@agent-device/kernel/errors'; import { successText } from '../../../utils/success-text.ts'; @@ -10,15 +15,19 @@ import { type BackendResultEnvelope, type RuntimeCommand, } from '../../runtime-types.ts'; -import { assertSupportedInteractionSurface, captureInteractionSnapshot } from './resolution.ts'; +import { + assertSupportedInteractionSurface, + captureInteractionSnapshot, + resolveInteractionTarget, +} from './resolution.ts'; import { resolveVisibleSnapshotViewport } from './viewport.ts'; export type GestureCommandOptions = CommandContext & { - gesture: GestureSemanticInput; + gesture: PublicGestureSemanticInput; }; export type GestureCommandResult = { - kind: GestureIntent; + kind: GestureIntent | 'drag'; durationMs: number; pointerCount: 1 | 2; from: Point; @@ -32,9 +41,13 @@ export const gestureCommand: RuntimeCommand { + const resolved = await resolveInteractionTarget( + runtime, + { + ...options, + target: target.startsWith('@') + ? { kind: 'ref', ref: target } + : { kind: 'selector', selector: target }, + }, + { + action: 'pan', + requireInteractive: false, + promoteToHittableAncestor: false, + }, + ); + if (!resolved.point) { + throw new AppError('COMMAND_FAILED', `gesture drag ${role} resolved without coordinates`); + } + return resolved.point; +} + async function captureGestureViewport( runtime: AgentDeviceRuntime, options: GestureCommandOptions, @@ -82,6 +146,10 @@ function centroidAt( }; } +function dragGestureMessage(input: DragGestureTargetInput): string { + return `Dragged ${input.source} to ${input.destination}`; +} + function gestureMessage(input: GestureSemanticInput, from: Point, to: Point): string { switch (input.intent) { case 'pan': { diff --git a/src/daemon/handlers/interaction-gesture.ts b/src/daemon/handlers/interaction-gesture.ts index 1bae0baad2..3f706514e3 100644 --- a/src/daemon/handlers/interaction-gesture.ts +++ b/src/daemon/handlers/interaction-gesture.ts @@ -10,7 +10,7 @@ import { SWIPE_REPETITION_MAX, SWIPE_SERIES_MAX_SCHEDULED_DURATION_MS, type GesturePayload, - type GestureSemanticInput, + type PublicGestureSemanticInput, type SwipePayload, } from '@agent-device/contracts/interaction'; import { AppError, normalizeError } from '@agent-device/kernel/errors'; @@ -45,21 +45,36 @@ export async function dispatchGestureViaRuntime( ): Promise { return await dispatchGestureInteraction(params, 'gesture', async (session) => { const input = readGesturePayload(params.req.input); - const normalized = normalizePublicGesture(input); - if (normalized.gesture.intent === 'pan' && params.req.internal?.gestureExecutionProfile) { - normalized.gesture.executionProfile = params.req.internal.gestureExecutionProfile; + const gesture: PublicGestureSemanticInput = + input.kind === 'drag' + ? { + intent: 'drag', + source: input.source, + destination: input.destination, + sourceHoldMs: input.sourceHoldMs, + moveMs: input.moveMs, + destinationHoldMs: input.destinationHoldMs, + } + : normalizePublicGesture(input).gesture; + if (gesture.intent === 'pan' && params.req.internal?.gestureExecutionProfile) { + gesture.executionProfile = params.req.internal.gestureExecutionProfile; } - requireGestureSupported(normalized.gesture, session.device); + requireGestureSupported( + gesture.intent === 'drag' + ? { intent: 'pan', origin: { x: 0, y: 0 }, delta: { x: 0, y: 0 } } + : gesture, + session.device, + ); const result = await createGestureRuntime(params).interactions.gesture({ session: params.sessionName, requestId: params.req.meta?.requestId, - gesture: normalized.gesture, + gesture, }); return { positionals: gesturePayloadToPositionals(input), flags: gestureReplayFlags(input, params.req.flags), responseData: gestureResponseData(result, { - executionProfile: resolveExecutionProfile(normalized.gesture), + executionProfile: resolveExecutionProfile(gesture), }), ...(input.kind === 'pinch' ? { recordingResultExtra: { scale: input.scale } } : {}), }; @@ -153,9 +168,10 @@ async function dispatchGestureInteraction( } } -function resolveExecutionProfile(gesture: GestureSemanticInput): string | undefined { +function resolveExecutionProfile(gesture: PublicGestureSemanticInput): string | undefined { if (gesture.intent === 'fling') return 'endpoint-hold'; if (gesture.intent === 'pan') return gesture.executionProfile ?? 'timed-pan'; + if (gesture.intent === 'drag') return 'hold-drag'; return undefined; }