From f2088bbed84e3f5953168c3f67b89be959e0e459 Mon Sep 17 00:00:00 2001 From: kua-agent Date: Tue, 1 Sep 2026 21:46:54 +0400 Subject: [PATCH] fix: picker first-mouse acceptance + event-based drag coords; in-process picker selftest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: RegionPickerView lacked acceptsFirstMouse — as an LSUIElement accessory app Shotdeck is never active when the hotkey fires, so the user's first click on the overlay was refused and the drag never started. Also plumbs the monitored event's location through the controller (hardware-cursor reads made the chain untestable). Adds PickerSelfTest (SHOTDECK_PICKER_SELFTEST): posts synthetic mouse events through the app's own queue, asserts the exact CaptureRegion, saves a mid-drag overlay bitmap. Coordinator ran it: PICKER-SELFTEST PASS rect=(200.0, 729.0, 400.0, 300.0); overlay bitmap shows dim+punch+chip. Co-Authored-By: Claude Fable 5 --- Sources/Shotdeck/PickerSelfTest.swift | 172 ++++++++++++++++++ Sources/Shotdeck/RegionPickerController.swift | 60 ++++-- Sources/Shotdeck/RegionPickerWindow.swift | 3 + Sources/Shotdeck/main.swift | 3 + 4 files changed, 219 insertions(+), 19 deletions(-) create mode 100644 Sources/Shotdeck/PickerSelfTest.swift diff --git a/Sources/Shotdeck/PickerSelfTest.swift b/Sources/Shotdeck/PickerSelfTest.swift new file mode 100644 index 0000000..f389d43 --- /dev/null +++ b/Sources/Shotdeck/PickerSelfTest.swift @@ -0,0 +1,172 @@ +import AppKit +import Darwin +import Foundation +import ShotdeckCore + +/// In-process self-test for the region picker, driven by `SHOTDECK_PICKER_SELFTEST`. +/// Posts synthetic mouse events through `NSApp.postEvent` only — never CGEventPost/taps. +@MainActor +enum PickerSelfTest { + private static let pointA = NSPoint(x: 200, y: 300) + private static let pointB = NSPoint(x: 600, y: 600) + private static let dragSteps = 6 + + /// Called from `main.swift` before `ShotdeckApp.main()`. Returns immediately when the + /// env var is unset; otherwise waits for launch, drives the production picker, and `exit`s. + static func runIfRequested() { + guard let raw = ProcessInfo.processInfo.environment["SHOTDECK_PICKER_SELFTEST"], + !raw.isEmpty + else { return } + + let output = URL(fileURLWithPath: raw, isDirectory: true) + // Hop onto a plain main-queue turn after NSApp starts. Nested run loops + // from a Swift Task do not drain NSApp's event queue. + DispatchQueue.main.async { + MainActor.assumeIsolated { + execute(outputDirectory: output) + } + } + } + + private static func execute(outputDirectory: URL) { + do { + try FileManager.default.createDirectory( + at: outputDirectory, + withIntermediateDirectories: true + ) + } catch { + fail(expected: expectedLabel(), got: "could not create output dir: \(error)") + } + + guard let screen = NSScreen.screens.first(where: { $0.frame.contains(pointA) }) + ?? NSScreen.screens.first + else { + fail(expected: expectedLabel(), got: "no NSScreen") + } + + let appKitRect = CGRect( + x: min(pointA.x, pointB.x), + y: min(pointA.y, pointB.y), + width: abs(pointB.x - pointA.x), + height: abs(pointB.y - pointA.y) + ).intersection(screen.frame) + let expected = CaptureRegion.fromAppKit(rect: appKitRect, on: screen) + + let picker = RegionPickerController() + var result: CaptureRegion?? + picker.pick { region in + result = .some(region) + } + + guard let overlay = overlayWindow(containing: pointA) else { + fail(expected: format(expected.rect), got: "no overlay window after pick()") + } + overlay.makeKey() + + var eventNumber = 1 + func post(_ type: NSEvent.EventType, at screenPoint: NSPoint, clickCount: Int) { + let locationInWindow = overlay.convertPoint(fromScreen: screenPoint) + guard let event = NSEvent.mouseEvent( + with: type, + location: locationInWindow, + modifierFlags: [], + timestamp: ProcessInfo.processInfo.systemUptime, + windowNumber: overlay.windowNumber, + context: nil, + eventNumber: eventNumber, + clickCount: clickCount, + pressure: 1 + ) else { + fail(expected: format(expected.rect), got: "NSEvent.mouseEvent(\(type.rawValue)) returned nil") + } + eventNumber += 1 + NSApp.postEvent(event, atStart: false) + // Local monitors run during NSApp.sendEvent (production dispatch), + // not during nextEvent dequeue. Drive that same path here. + NSApp.sendEvent(event) + } + + post(.leftMouseDown, at: pointA, clickCount: 1) + for step in 1...(dragSteps / 2) { + post(.leftMouseDragged, at: interpolate(step), clickCount: 0) + } + + writeOverlayBitmap(overlay, to: outputDirectory.appendingPathComponent("overlay-middrag.png")) + + for step in ((dragSteps / 2) + 1)...dragSteps { + post(.leftMouseDragged, at: interpolate(step), clickCount: 0) + } + post(.leftMouseUp, at: pointB, clickCount: 1) + + guard let wrapped = result else { + fail(expected: format(expected.rect), got: "completion never fired") + } + guard let got = wrapped else { + fail(expected: format(expected.rect), got: "nil") + } + + if got.rect != expected.rect { + fail(expected: format(expected.rect), got: format(got.rect)) + } + + print("PICKER-SELFTEST PASS rect=\(format(got.rect))") + fflush(stdout) + exit(0) + } + + private static func interpolate(_ step: Int) -> NSPoint { + let t = CGFloat(step) / CGFloat(dragSteps) + return NSPoint( + x: pointA.x + (pointB.x - pointA.x) * t, + y: pointA.y + (pointB.y - pointA.y) * t + ) + } + + private static func overlayWindow(containing point: NSPoint) -> RegionPickerWindow? { + let overlays = NSApp.windows.compactMap { $0 as? RegionPickerWindow } + return overlays.first(where: { $0.coveringScreen.frame.contains(point) }) ?? overlays.first + } + + private static func writeOverlayBitmap(_ overlay: RegionPickerWindow, to url: URL) { + guard let view = overlay.contentView else { + fail(expected: expectedLabel(), got: "overlay has no contentView") + } + overlay.layoutIfNeeded() + view.layoutSubtreeIfNeeded() + view.display() + let bounds = view.bounds + guard let rep = view.bitmapImageRepForCachingDisplay(in: bounds) else { + fail(expected: expectedLabel(), got: "bitmapImageRepForCachingDisplay failed") + } + view.cacheDisplay(in: bounds, to: rep) + guard let png = rep.representation(using: .png, properties: [:]) else { + fail(expected: expectedLabel(), got: "PNG encode failed") + } + do { + try png.write(to: url) + } catch { + fail(expected: expectedLabel(), got: "could not write \(url.path): \(error)") + } + print(url.path) + fflush(stdout) + } + + private static func expectedLabel() -> String { + format(CGRect( + x: min(pointA.x, pointB.x), + y: min(pointA.y, pointB.y), + width: abs(pointB.x - pointA.x), + height: abs(pointB.y - pointA.y) + )) + } + + private static func format(_ rect: CGRect) -> String { + "(\(rect.origin.x), \(rect.origin.y), \(rect.width), \(rect.height))" + } + + private static func fail(expected: String, got: String) -> Never { + print("PICKER-SELFTEST FAIL expected=\(expected) got=\(got)") + fflush(stdout) + exit(1) + } +} diff --git a/Sources/Shotdeck/RegionPickerController.swift b/Sources/Shotdeck/RegionPickerController.swift index 88d32bb..114fdcf 100644 --- a/Sources/Shotdeck/RegionPickerController.swift +++ b/Sources/Shotdeck/RegionPickerController.swift @@ -48,20 +48,27 @@ public final class RegionPickerController { matching: [.leftMouseDown, .leftMouseDragged, .leftMouseUp, .keyDown] ) { [weak self] event in guard let self else { return event } - // NSEvent is not Sendable; lift the two Sendable fields the handler - // needs. Mouse geometry is read from NSEvent.mouseLocation on the - // main thread inside the isolated methods, per spec. + // NSEvent is not Sendable; lift Sendable fields the handler needs. + // Prefer the event's window-local point converted to global AppKit + // coordinates; NSEvent.mouseLocation is only a fallback. + // keyCode is only valid on key events — reading it on a mouse event raises. let type = event.type - let keyCode = event.keyCode + let keyCode: UInt16 = (type == .keyDown) ? event.keyCode : 0 + let locationInWindow = event.locationInWindow + let windowNumber = event.windowNumber let consume = MainActor.assumeIsolated { - self.handle(type: type, keyCode: keyCode) + let location = self.globalAppKitLocation( + locationInWindow: locationInWindow, + windowNumber: windowNumber + ) + return self.handle(type: type, keyCode: keyCode, location: location) } return consume ? nil : event } } /// Returns `true` when the event should be swallowed. - private func handle(type: NSEvent.EventType, keyCode: UInt16) -> Bool { + private func handle(type: NSEvent.EventType, keyCode: UInt16, location: NSPoint?) -> Bool { switch type { case .keyDown: if keyCode == 53 { // kVK_Escape @@ -70,21 +77,21 @@ public final class RegionPickerController { } return false case .leftMouseDown: - handleMouseDown() + handleMouseDown(location: location) return false case .leftMouseDragged: - handleMouseDragged() + handleMouseDragged(location: location) return false case .leftMouseUp: - handleMouseUp() + handleMouseUp(location: location) return false default: return false } } - private func handleMouseDown() { - let point = NSEvent.mouseLocation + private func handleMouseDown(location: NSPoint?) { + let point = location ?? NSEvent.mouseLocation dragStart = point startScreen = NSScreen.screens.first(where: { $0.frame.contains(point) }) ?? NSScreen.screens.first @@ -99,17 +106,17 @@ public final class RegionPickerController { } } - private func handleMouseDragged() { + private func handleMouseDragged(location: NSPoint?) { guard dragStart != nil, startScreen != nil else { return } - applyLiveSelection() + applyLiveSelection(current: location) } - private func handleMouseUp() { + private func handleMouseUp(location: NSPoint?) { guard let startScreen else { dragStart = nil return } - guard let rect = currentClampedRect(), rect.width >= 8, rect.height >= 8 else { + guard let rect = currentClampedRect(current: location), rect.width >= 8, rect.height >= 8 else { // Mis-click: reset drag state, leave every window open and fully dimmed. dragStart = nil self.startScreen = nil @@ -122,8 +129,8 @@ public final class RegionPickerController { finish(region: region) } - private func applyLiveSelection() { - guard let startScreen, let rect = currentClampedRect() else { return } + private func applyLiveSelection(current: NSPoint?) { + guard let startScreen, let rect = currentClampedRect(current: current) else { return } let local = Self.localRect(from: rect, on: startScreen) let text = "\(Int(rect.width)) x \(Int(rect.height))" window(for: startScreen)?.updateSelection(localRect: local, sizeText: text) @@ -134,9 +141,9 @@ public final class RegionPickerController { /// Normalize the drag (so bottom-right → up-left is not misjudged) then clamp /// to the screen the gesture started on — never selects across displays. - private func currentClampedRect() -> CGRect? { + private func currentClampedRect(current: NSPoint?) -> CGRect? { guard let dragStart, let startScreen else { return nil } - let current = NSEvent.mouseLocation + let current = current ?? NSEvent.mouseLocation let normalized = CGRect( x: min(dragStart.x, current.x), y: min(dragStart.y, current.y), @@ -146,6 +153,21 @@ public final class RegionPickerController { return normalized.intersection(startScreen.frame) } + /// Convert a monitored event's `locationInWindow` into global AppKit coordinates + /// (bottom-left origin, matching `NSEvent.mouseLocation` / `NSScreen.frame`). + private func globalAppKitLocation(locationInWindow: NSPoint, windowNumber: Int) -> NSPoint? { + if let window = windows.first(where: { $0.windowNumber == windowNumber }) { + return window.convertPoint(toScreen: locationInWindow) + } + if windowNumber != 0, let window = NSApp.window(withWindowNumber: windowNumber) { + return window.convertPoint(toScreen: locationInWindow) + } + if windowNumber == 0 { + return locationInWindow + } + return nil + } + private func window(for screen: NSScreen) -> RegionPickerWindow? { windows.first { $0.coveringScreen === screen } } diff --git a/Sources/Shotdeck/RegionPickerWindow.swift b/Sources/Shotdeck/RegionPickerWindow.swift index a5c315d..5ffaa3c 100644 --- a/Sources/Shotdeck/RegionPickerWindow.swift +++ b/Sources/Shotdeck/RegionPickerWindow.swift @@ -29,6 +29,7 @@ final class RegionPickerWindow: NSPanel { becomesKeyOnlyIfNeeded = false animationBehavior = .none collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary, .stationary] + acceptsMouseMovedEvents = true contentView = pickerView } @@ -53,6 +54,8 @@ private final class RegionPickerView: NSView { override var isOpaque: Bool { false } override var acceptsFirstResponder: Bool { true } + // Accessory-app trap: first click on an inactive overlay is first-mouse and is dropped unless accepted. + override func acceptsFirstMouse(for event: NSEvent?) -> Bool { true } override func draw(_ dirtyRect: NSRect) { super.draw(dirtyRect) diff --git a/Sources/Shotdeck/main.swift b/Sources/Shotdeck/main.swift index 21ce466..81caac8 100644 --- a/Sources/Shotdeck/main.swift +++ b/Sources/Shotdeck/main.swift @@ -7,6 +7,9 @@ import ShotdeckCore if ProcessInfo.processInfo.environment["SHOTDECK_SNAPSHOT_DIR"] != nil { MainActor.assumeIsolated { PanelSnapshot.runIfRequested() } } +if ProcessInfo.processInfo.environment["SHOTDECK_PICKER_SELFTEST"] != nil { + MainActor.assumeIsolated { PickerSelfTest.runIfRequested() } +} ShotdeckApp.main() struct ShotdeckApp: App {