Window hints

ViMac-Swift/Accessibility/HintMode/HintModeQueryService.swift

@@ -8,15 +8,18 @@
 
 import Cocoa
 import RxSwift
+import AXSwift
 
 class HintModeQueryService {
     let app: NSRunningApplication?
     let window: Element?
+    let menu: Element?
     let hintCharacters: String
 
-    init(app: NSRunningApplication?, window: Element?, hintCharacters: String) {
+    init(app: NSRunningApplication?, window: Element?, menu: Element?, hintCharacters: String) {
         self.app = app
         self.window = window
+        self.menu = menu
         self.hintCharacters = hintCharacters
     }
 
@@ -40,6 +43,15 @@ class HintModeQueryService {
             menuBarElements = Utils.singleToObservable(single: queryMenuBarSingle(app: app))
         }
 
+        if let menu = menu {
+            return Utils.eagerConcat(observables: [
+                menuBarElements,
+                Utils.singleToObservable(single: queryMenuBarExtrasSingle()),
+                Utils.singleToObservable(single: queryNotificationCenterSingle()),
+                Utils.singleToObservable(single: queryOpenedMenuSingle(menu: menu))
+            ])
+        }
+
         var windowElements = nothing
         if let app = app,
            let window = window {
@@ -68,6 +80,26 @@ class HintModeQueryService {
         })
     }
 
+    private func queryOpenedMenuSingle(menu: Element) -> Single<[Element]> {
+        return Single.create(subscribe: { event in
+            let thread = Thread.init(block: {
+                print(menu.role)
+                let menuItemsOptional: [AXUIElement]? = try? UIElement(menu.rawElement).attribute(.children)
+                print(menuItemsOptional?.count)
+                let menuItems = menuItemsOptional ?? []
+                let menuItemElements = menuItems
+                    .map { Element.initialize(rawElement: $0) }
+                    .compactMap({ $0 })
+                print(menuItemElements)
+                event(.success(menuItemElements))
+            })
+            thread.start()
+            return Disposables.create {
+                thread.cancel()
+            }
+        })
+    }
+
     private func queryMenuBarSingle(app: NSRunningApplication) -> Single<[Element]> {
         return Single.create(subscribe: { event in
             let thread = Thread.init(block: {                

ViMac-Swift/Activation/HoldKeyListener.swift

@@ -30,7 +30,7 @@ class HoldKeyListener {
     func start() {
         if eventTap == nil {
             let mask = CGEventMask(1 << CGEventType.keyDown.rawValue | 1 << CGEventType.keyUp.rawValue)
-            eventTap = GlobalEventTap(eventMask: mask, onEvent: { [weak self] event -> CGEvent? in
+            eventTap = GlobalEventTap(eventMask: mask, placement: .headInsertEventTap, onEvent: { [weak self] event -> CGEvent? in
                 guard let self = self else { return event }
 
                 return self.onEvent(event: event)

ViMac-Swift/AppDelegate.swift

@@ -237,6 +237,20 @@ class AppDelegate: NSObject, NSApplicationDelegate {
                 self.modeCoordinator.deactivate()
             })
         )
+
+        self.frontmostAppService.observeMenuOpened({ [weak self] menu in
+            guard let self = self else { return }
+
+            self.modeCoordinator.openedMenu = menu
+        })
+
+        self.frontmostAppService.observeMenuClosed({ [weak self] menu in
+            guard let self = self else { return }
+
+            if self.modeCoordinator.openedMenu == menu {
+                self.modeCoordinator.openedMenu = nil
+            }
+        })
 
         _ = self.compositeDisposable.insert(hintModeShortcutObservable
             .observeOn(MainScheduler.instance)

ViMac-Swift/GlobalEventTap.swift

@@ -12,14 +12,16 @@ import os
 
 class GlobalEventTap {
     let eventMask: CGEventMask
+    let placement: CGEventTapPlacement
     let eventHandler: (CGEvent) -> CGEvent?
 
     var runLoopSource: CFRunLoopSource?
     var eventTap: CFMachPort?
     var selfPtr: Unmanaged<GlobalEventTap>!
 
-    init(eventMask: CGEventMask, onEvent: @escaping (CGEvent) -> CGEvent?) {
+    init(eventMask: CGEventMask, placement: CGEventTapPlacement, onEvent: @escaping (CGEvent) -> CGEvent?) {
         self.eventMask = eventMask
+        self.placement = placement
         self.eventHandler = onEvent
         selfPtr = Unmanaged.passRetained(self)
     }
@@ -92,7 +94,7 @@ class GlobalEventTap {
     }
 
     private func createEventTap() -> CFMachPort? {
-        let eventTap = CGEvent.tapCreate(tap: .cgSessionEventTap, place: .headInsertEventTap, options: .defaultTap, eventsOfInterest: eventMask, callback: { proxy, type, event, refcon in
+        let eventTap = CGEvent.tapCreate(tap: .cgSessionEventTap, place: placement, options: .defaultTap, eventsOfInterest: eventMask, callback: { proxy, type, event, refcon in
             // Trick from https://stackoverflow.com/questions/33260808/how-to-use-instance-method-as-callback-for-function-which-takes-only-func-or-lit
             let mySelf = Unmanaged<GlobalEventTap>.fromOpaque(refcon!).takeUnretainedValue()
             return mySelf.eventTapCallback(proxy: proxy, type: type, event: event, refcon: refcon)

ViMac-Swift/HideCursorGlobally.h

@@ -13,6 +13,7 @@ NS_ASSUME_NONNULL_BEGIN
 @interface HideCursorGlobally : NSObject
 + (void) hide;
 + (void) unhide;
++ (void) _activateWindow: (pid_t) pid;
 @end
 
 NS_ASSUME_NONNULL_END

ViMac-Swift/HideCursorGlobally.m

@@ -26,4 +26,11 @@ + (void) hide {
 + (void) unhide {
     CGDisplayShowCursor(CGMainDisplayID());
 }
+
+// this should probably belong in its own file.
++ (void) _activateWindow: (pid_t) pid {
+    ProcessSerialNumber psn;
+    GetProcessForPID(pid, &psn);
+    SetFrontProcessWithOptions(&psn, kSetFrontProcessFrontWindowOnly);
+}
 @end

ViMac-Swift/HintView.swift

@@ -10,10 +10,10 @@ import Cocoa
 import AXSwift
 
 class HintView: NSView {
-    static let borderColor = NSColor.darkGray
-    static let backgroundColor = NSColor(red: 255 / 255, green: 224 / 255, blue: 112 / 255, alpha: 1)
-    static let untypedHintColor = NSColor.black
-    static let typedHintColor = NSColor(red: 212 / 255, green: 172 / 255, blue: 58 / 255, alpha: 1)
+    let borderColor = NSColor.darkGray
+    let backgroundColor = NSColor(red: 255 / 255, green: 224 / 255, blue: 112 / 255, alpha: 1)
+    let untypedHintColor = NSColor.black
+    let typedHintColor = NSColor(red: 212 / 255, green: 172 / 255, blue: 58 / 255, alpha: 1)
 
     let associatedElement: Element
     var hintTextView: HintText?
@@ -25,16 +25,16 @@ class HintView: NSView {
         self.associatedElement = associatedElement
         super.init(frame: .zero)
 
-        self.hintTextView = HintText(hintTextSize: hintTextSize, hintText: hintText, typedHintText: typedHintText)
+        self.hintTextView = HintText(hintTextSize: hintTextSize, hintText: hintText, typedHintText: typedHintText, untypedHintColor: untypedHintColor, typedHintColor: typedHintColor)
         self.subviews.append(hintTextView!)
 
         self.wantsLayer = true
 
 
         self.layer?.borderWidth = borderWidth
 
-        self.layer?.backgroundColor = HintView.backgroundColor.cgColor
-        self.layer?.borderColor = HintView.borderColor.cgColor
+        self.layer?.backgroundColor = backgroundColor.cgColor
+        self.layer?.borderColor = borderColor.cgColor
         self.layer?.cornerRadius = cornerRadius
 
         self.translatesAutoresizingMaskIntoConstraints = false
@@ -68,10 +68,71 @@ class HintView: NSView {
             height: height()
         )
     }
+
+    func updateTypedText(typed: String) {
+        self.hintTextView!.updateTypedText(typed: typed)
+    }
+}
+
+class WindowHintView: NSView {
+    let borderColor = NSColor.darkGray
+    let backgroundColor = NSColor(red: 25 / 255, green: 25 / 255, blue: 25 / 255, alpha: 1)
+    let untypedHintColor = NSColor.white
+    let typedHintColor = NSColor.darkGray
+
+    let associatedElement: Element
+    var hintTextView: HintText?
+
+    let borderWidth: CGFloat = 1.0
+    let cornerRadius: CGFloat = 3.0
+    let hintTextSize: CGFloat = 40
 
-    func addHintText(hintTextSize: CGFloat, hintText: String, typedHintText: String) {
-        self.hintTextView = HintText(hintTextSize: hintTextSize, hintText: hintText, typedHintText: typedHintText)
+    required init(associatedElement: Element, hintText: String, typedHintText: String) {
+        self.associatedElement = associatedElement
+        super.init(frame: .zero)
+
+        self.hintTextView = HintText(hintTextSize: hintTextSize, hintText: hintText, typedHintText: typedHintText, untypedHintColor: untypedHintColor, typedHintColor: typedHintColor)
         self.subviews.append(hintTextView!)
+
+        self.wantsLayer = true
+
+
+        self.layer?.borderWidth = borderWidth
+
+        self.layer?.backgroundColor = backgroundColor.cgColor
+        self.layer?.borderColor = borderColor.cgColor
+        self.layer?.cornerRadius = cornerRadius
+
+        self.translatesAutoresizingMaskIntoConstraints = false
+
+        self.hintTextView!.centerYAnchor.constraint(equalTo: self.centerYAnchor).isActive = true
+        self.hintTextView!.centerXAnchor.constraint(equalTo: self.centerXAnchor).isActive = true
+
+        self.widthAnchor.constraint(equalToConstant: width()).isActive = true
+        self.heightAnchor.constraint(equalToConstant: height()).isActive = true
+    }
+
+    private func width() -> CGFloat {
+        return self.hintTextView!.intrinsicContentSize.width + 2 * borderWidth
+    }
+
+    private func height() -> CGFloat {
+        self.hintTextView!.intrinsicContentSize.height + 2 * borderWidth
+    }
+
+    required init?(coder: NSCoder) {
+        fatalError()
+    }
+
+    override init(frame frameRect: NSRect) {
+        fatalError()
+    }
+
+    override var intrinsicContentSize: NSSize {
+        return .init(
+            width: width(),
+            height: height()
+        )
     }
 
     func updateTypedText(typed: String) {
@@ -80,21 +141,29 @@ class HintView: NSView {
 }
 
 class HintText: NSTextField {
+    let hintText: String
+    let hintTextSize: CGFloat
+    let untypedHintColor: NSColor
+    let typedHintColor: NSColor
 
-    required init(hintTextSize: CGFloat, hintText: String, typedHintText: String) {
+    required init(hintTextSize: CGFloat, hintText: String, typedHintText: String, untypedHintColor: NSColor, typedHintColor: NSColor) {
+        self.hintText = hintText
+        self.hintTextSize = hintTextSize
+        self.untypedHintColor = untypedHintColor
+        self.typedHintColor = typedHintColor
         super.init(frame: .zero)
         self.translatesAutoresizingMaskIntoConstraints = false
-        self.setup(hintTextSize: hintTextSize, hintText: hintText, typedHintText: typedHintText)
+        self.setup()
     }
 
     required init(coder: NSCoder) {
         fatalError()
     }
 
-    func setup(hintTextSize: CGFloat, hintText: String, typedHintText: String) {
+    func setup() {
         self.stringValue = hintText
         self.font = NSFont.systemFont(ofSize: hintTextSize, weight: .bold)
-        self.textColor = .black
+        self.textColor = untypedHintColor
 
         // isBezeled causes unwanted padding.
         self.isBezeled = false
@@ -114,10 +183,10 @@ class HintText: NSTextField {
         let hintText = self.attributedStringValue.string
         let attr = NSMutableAttributedString(string: hintText)
         let range = NSMakeRange(0, hintText.count)
-        attr.addAttributes([NSAttributedString.Key.foregroundColor : HintView.untypedHintColor], range: range)
+        attr.addAttributes([NSAttributedString.Key.foregroundColor : untypedHintColor], range: range)
         if hintText.lowercased().starts(with: typed.lowercased()) {
             let typedRange = NSMakeRange(0, typed.count)
-            attr.addAttributes([NSAttributedString.Key.foregroundColor : HintView.typedHintColor], range: typedRange)
+            attr.addAttributes([NSAttributedString.Key.foregroundColor : typedHintColor], range: typedRange)
         }
         self.attributedStringValue = attr
     }

ViMac-Swift/KeySequenceListener.swift

@@ -56,7 +56,7 @@ class KeySequenceListener {
 
     func start() {
         if eventTap == nil {
-            eventTap = GlobalEventTap(eventMask: mask, onEvent: { [weak self] event -> CGEvent? in
+            eventTap = GlobalEventTap(eventMask: mask, placement: .headInsertEventTap, onEvent: { [weak self] event -> CGEvent? in
                 guard let self = self else { return event}
                 return self.onEvent(event: event)
             })

ViMac-Swift/ModeCoordinator.swift

@@ -25,6 +25,7 @@ class ModeCoordinator: ModeControllerDelegate {
     let hintModeKeySequence: [Character] = ["f", "d"]
     private let keySequenceListener: VimacKeySequenceListener
     private var holdKeyListener: HoldKeyListener?
+    var openedMenu: AXUIElement?
 
     var modeController: ModeController?
 
@@ -79,6 +80,7 @@ class ModeCoordinator: ModeControllerDelegate {
 
     func modeDeactivated(controller: ModeController) {
         self.modeController = nil
+        self.openedMenu = nil
 
         if self.forceKBLayout != nil {
             self.priorKBLayout?.select()
@@ -130,12 +132,13 @@ class ModeCoordinator: ModeControllerDelegate {
         }
 
         let app = NSWorkspace.shared.frontmostApplication
+        let openedMenu = openedMenuElement()
         let window = app.flatMap { focusedWindow(app: $0) }
 
         if let app = app {
             // the app crashes when talking to its own accessibility server
             let isTargetVimac = app.bundleIdentifier == Bundle.main.bundleIdentifier
-            if isTargetVimac {
+        if isTargetVimac {
                 return
             }
         }
@@ -144,13 +147,14 @@ class ModeCoordinator: ModeControllerDelegate {
 
         Analytics.shared().track("Hint Mode Activated", properties: [
             "Target Application": app?.bundleIdentifier as Any,
-            "Activation Mechanism": mechanism
+            "Activation Mechanism": mechanism,
+            "Root Element Role": openedMenu?.role ?? window?.role
         ])
 
         let activationCount = UserDefaults.standard.integer(forKey: "hintModeActivationCount")
         UserDefaults.standard.set(activationCount + 1, forKey: "hintModeActivationCount")
 
-        modeController = HintModeController(app: app, window: window)
+        modeController = HintModeController(app: app, window: window, menu: openedMenu)
         modeController?.delegate = self
         modeController!.activate()
     }
@@ -167,6 +171,13 @@ class ModeCoordinator: ModeControllerDelegate {
         return observation
     }
 
+    func openedMenuElement() -> Element? {
+        guard let e = openedMenu else { return nil }
+
+        // in addition to querying for useful attributes, it also tests for death of opened menu since it may no longer exist
+        return Element.initialize(rawElement: e)
+    }
+
     // fun fact, focusedWindow need not return "AXWindow"...
     private func focusedWindow(app: NSRunningApplication) -> Element? {
         let axAppOptional = Application.init(app)