fix: picker first-mouse acceptance + event-based drag coords + in-process selftest #14

Merged
kua-agent merged 1 commits from fix/picker-first-mouse-20260901 into feat/shotdeck-20260830 2026-09-01 17:47:09 +00:00
4 changed files with 219 additions and 19 deletions
+172
View File
@@ -0,0 +1,172 @@
import AppKit
import Darwin
import Foundation
import ShotdeckCore
/// In-process self-test for the region picker, driven by `SHOTDECK_PICKER_SELFTEST`.
/// Posts synthetic mouse events through `NSApp.postEvent` only never CGEventPost/taps.
@MainActor
enum PickerSelfTest {
private static let pointA = NSPoint(x: 200, y: 300)
private static let pointB = NSPoint(x: 600, y: 600)
private static let dragSteps = 6
/// Called from `main.swift` before `ShotdeckApp.main()`. Returns immediately when the
/// env var is unset; otherwise waits for launch, drives the production picker, and `exit`s.
static func runIfRequested() {
guard let raw = ProcessInfo.processInfo.environment["SHOTDECK_PICKER_SELFTEST"],
!raw.isEmpty
else { return }
let output = URL(fileURLWithPath: raw, isDirectory: true)
// Hop onto a plain main-queue turn after NSApp starts. Nested run loops
// from a Swift Task do not drain NSApp's event queue.
DispatchQueue.main.async {
MainActor.assumeIsolated {
execute(outputDirectory: output)
}
}
}
private static func execute(outputDirectory: URL) {
do {
try FileManager.default.createDirectory(
at: outputDirectory,
withIntermediateDirectories: true
)
} catch {
fail(expected: expectedLabel(), got: "could not create output dir: \(error)")
}
guard let screen = NSScreen.screens.first(where: { $0.frame.contains(pointA) })
?? NSScreen.screens.first
else {
fail(expected: expectedLabel(), got: "no NSScreen")
}
let appKitRect = CGRect(
x: min(pointA.x, pointB.x),
y: min(pointA.y, pointB.y),
width: abs(pointB.x - pointA.x),
height: abs(pointB.y - pointA.y)
).intersection(screen.frame)
let expected = CaptureRegion.fromAppKit(rect: appKitRect, on: screen)
let picker = RegionPickerController()
var result: CaptureRegion??
picker.pick { region in
result = .some(region)
}
guard let overlay = overlayWindow(containing: pointA) else {
fail(expected: format(expected.rect), got: "no overlay window after pick()")
}
overlay.makeKey()
var eventNumber = 1
func post(_ type: NSEvent.EventType, at screenPoint: NSPoint, clickCount: Int) {
let locationInWindow = overlay.convertPoint(fromScreen: screenPoint)
guard let event = NSEvent.mouseEvent(
with: type,
location: locationInWindow,
modifierFlags: [],
timestamp: ProcessInfo.processInfo.systemUptime,
windowNumber: overlay.windowNumber,
context: nil,
eventNumber: eventNumber,
clickCount: clickCount,
pressure: 1
) else {
fail(expected: format(expected.rect), got: "NSEvent.mouseEvent(\(type.rawValue)) returned nil")
}
eventNumber += 1
NSApp.postEvent(event, atStart: false)
// Local monitors run during NSApp.sendEvent (production dispatch),
// not during nextEvent dequeue. Drive that same path here.
NSApp.sendEvent(event)
}
post(.leftMouseDown, at: pointA, clickCount: 1)
for step in 1...(dragSteps / 2) {
post(.leftMouseDragged, at: interpolate(step), clickCount: 0)
}
writeOverlayBitmap(overlay, to: outputDirectory.appendingPathComponent("overlay-middrag.png"))
for step in ((dragSteps / 2) + 1)...dragSteps {
post(.leftMouseDragged, at: interpolate(step), clickCount: 0)
}
post(.leftMouseUp, at: pointB, clickCount: 1)
guard let wrapped = result else {
fail(expected: format(expected.rect), got: "completion never fired")
}
guard let got = wrapped else {
fail(expected: format(expected.rect), got: "nil")
}
if got.rect != expected.rect {
fail(expected: format(expected.rect), got: format(got.rect))
}
print("PICKER-SELFTEST PASS rect=\(format(got.rect))")
fflush(stdout)
exit(0)
}
private static func interpolate(_ step: Int) -> NSPoint {
let t = CGFloat(step) / CGFloat(dragSteps)
return NSPoint(
x: pointA.x + (pointB.x - pointA.x) * t,
y: pointA.y + (pointB.y - pointA.y) * t
)
}
private static func overlayWindow(containing point: NSPoint) -> RegionPickerWindow? {
let overlays = NSApp.windows.compactMap { $0 as? RegionPickerWindow }
return overlays.first(where: { $0.coveringScreen.frame.contains(point) }) ?? overlays.first
}
private static func writeOverlayBitmap(_ overlay: RegionPickerWindow, to url: URL) {
guard let view = overlay.contentView else {
fail(expected: expectedLabel(), got: "overlay has no contentView")
}
overlay.layoutIfNeeded()
view.layoutSubtreeIfNeeded()
view.display()
let bounds = view.bounds
guard let rep = view.bitmapImageRepForCachingDisplay(in: bounds) else {
fail(expected: expectedLabel(), got: "bitmapImageRepForCachingDisplay failed")
}
view.cacheDisplay(in: bounds, to: rep)
guard let png = rep.representation(using: .png, properties: [:]) else {
fail(expected: expectedLabel(), got: "PNG encode failed")
}
do {
try png.write(to: url)
} catch {
fail(expected: expectedLabel(), got: "could not write \(url.path): \(error)")
}
print(url.path)
fflush(stdout)
}
private static func expectedLabel() -> String {
format(CGRect(
x: min(pointA.x, pointB.x),
y: min(pointA.y, pointB.y),
width: abs(pointB.x - pointA.x),
height: abs(pointB.y - pointA.y)
))
}
private static func format(_ rect: CGRect) -> String {
"(\(rect.origin.x), \(rect.origin.y), \(rect.width), \(rect.height))"
}
private static func fail(expected: String, got: String) -> Never {
print("PICKER-SELFTEST FAIL expected=\(expected) got=\(got)")
fflush(stdout)
exit(1)
}
}
+41 -19
View File
@@ -48,20 +48,27 @@ public final class RegionPickerController {
matching: [.leftMouseDown, .leftMouseDragged, .leftMouseUp, .keyDown]
) { [weak self] event in
guard let self else { return event }
// NSEvent is not Sendable; lift the two Sendable fields the handler
// needs. Mouse geometry is read from NSEvent.mouseLocation on the
// main thread inside the isolated methods, per spec.
// NSEvent is not Sendable; lift Sendable fields the handler needs.
// Prefer the event's window-local point converted to global AppKit
// coordinates; NSEvent.mouseLocation is only a fallback.
// keyCode is only valid on key events reading it on a mouse event raises.
let type = event.type
let keyCode = event.keyCode
let keyCode: UInt16 = (type == .keyDown) ? event.keyCode : 0
let locationInWindow = event.locationInWindow
let windowNumber = event.windowNumber
let consume = MainActor.assumeIsolated {
self.handle(type: type, keyCode: keyCode)
let location = self.globalAppKitLocation(
locationInWindow: locationInWindow,
windowNumber: windowNumber
)
return self.handle(type: type, keyCode: keyCode, location: location)
}
return consume ? nil : event
}
}
/// Returns `true` when the event should be swallowed.
private func handle(type: NSEvent.EventType, keyCode: UInt16) -> Bool {
private func handle(type: NSEvent.EventType, keyCode: UInt16, location: NSPoint?) -> Bool {
switch type {
case .keyDown:
if keyCode == 53 { // kVK_Escape
@@ -70,21 +77,21 @@ public final class RegionPickerController {
}
return false
case .leftMouseDown:
handleMouseDown()
handleMouseDown(location: location)
return false
case .leftMouseDragged:
handleMouseDragged()
handleMouseDragged(location: location)
return false
case .leftMouseUp:
handleMouseUp()
handleMouseUp(location: location)
return false
default:
return false
}
}
private func handleMouseDown() {
let point = NSEvent.mouseLocation
private func handleMouseDown(location: NSPoint?) {
let point = location ?? NSEvent.mouseLocation
dragStart = point
startScreen = NSScreen.screens.first(where: { $0.frame.contains(point) })
?? NSScreen.screens.first
@@ -99,17 +106,17 @@ public final class RegionPickerController {
}
}
private func handleMouseDragged() {
private func handleMouseDragged(location: NSPoint?) {
guard dragStart != nil, startScreen != nil else { return }
applyLiveSelection()
applyLiveSelection(current: location)
}
private func handleMouseUp() {
private func handleMouseUp(location: NSPoint?) {
guard let startScreen else {
dragStart = nil
return
}
guard let rect = currentClampedRect(), rect.width >= 8, rect.height >= 8 else {
guard let rect = currentClampedRect(current: location), rect.width >= 8, rect.height >= 8 else {
// Mis-click: reset drag state, leave every window open and fully dimmed.
dragStart = nil
self.startScreen = nil
@@ -122,8 +129,8 @@ public final class RegionPickerController {
finish(region: region)
}
private func applyLiveSelection() {
guard let startScreen, let rect = currentClampedRect() else { return }
private func applyLiveSelection(current: NSPoint?) {
guard let startScreen, let rect = currentClampedRect(current: current) else { return }
let local = Self.localRect(from: rect, on: startScreen)
let text = "\(Int(rect.width)) x \(Int(rect.height))"
window(for: startScreen)?.updateSelection(localRect: local, sizeText: text)
@@ -134,9 +141,9 @@ public final class RegionPickerController {
/// Normalize the drag (so bottom-right up-left is not misjudged) then clamp
/// to the screen the gesture started on never selects across displays.
private func currentClampedRect() -> CGRect? {
private func currentClampedRect(current: NSPoint?) -> CGRect? {
guard let dragStart, let startScreen else { return nil }
let current = NSEvent.mouseLocation
let current = current ?? NSEvent.mouseLocation
let normalized = CGRect(
x: min(dragStart.x, current.x),
y: min(dragStart.y, current.y),
@@ -146,6 +153,21 @@ public final class RegionPickerController {
return normalized.intersection(startScreen.frame)
}
/// Convert a monitored event's `locationInWindow` into global AppKit coordinates
/// (bottom-left origin, matching `NSEvent.mouseLocation` / `NSScreen.frame`).
private func globalAppKitLocation(locationInWindow: NSPoint, windowNumber: Int) -> NSPoint? {
if let window = windows.first(where: { $0.windowNumber == windowNumber }) {
return window.convertPoint(toScreen: locationInWindow)
}
if windowNumber != 0, let window = NSApp.window(withWindowNumber: windowNumber) {
return window.convertPoint(toScreen: locationInWindow)
}
if windowNumber == 0 {
return locationInWindow
}
return nil
}
private func window(for screen: NSScreen) -> RegionPickerWindow? {
windows.first { $0.coveringScreen === screen }
}
@@ -29,6 +29,7 @@ final class RegionPickerWindow: NSPanel {
becomesKeyOnlyIfNeeded = false
animationBehavior = .none
collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary, .stationary]
acceptsMouseMovedEvents = true
contentView = pickerView
}
@@ -53,6 +54,8 @@ private final class RegionPickerView: NSView {
override var isOpaque: Bool { false }
override var acceptsFirstResponder: Bool { true }
// Accessory-app trap: first click on an inactive overlay is first-mouse and is dropped unless accepted.
override func acceptsFirstMouse(for event: NSEvent?) -> Bool { true }
override func draw(_ dirtyRect: NSRect) {
super.draw(dirtyRect)
+3
View File
@@ -7,6 +7,9 @@ import ShotdeckCore
if ProcessInfo.processInfo.environment["SHOTDECK_SNAPSHOT_DIR"] != nil {
MainActor.assumeIsolated { PanelSnapshot.runIfRequested() }
}
if ProcessInfo.processInfo.environment["SHOTDECK_PICKER_SELFTEST"] != nil {
MainActor.assumeIsolated { PickerSelfTest.runIfRequested() }
}
ShotdeckApp.main()
struct ShotdeckApp: App {