From d8f2ebcaeadcb64801151b47a11c108dd8cf2cfc Mon Sep 17 00:00:00 2001 From: Exidex <16986685+exidex@users.noreply.github.com> Date: Thu, 2 Oct 2025 18:54:36 +0200 Subject: [PATCH] not gonna comback to this --- rust/client/build.rs | 2 +- .../src/ui/window_tracking/macos/apps.rs | 24 +++- .../src/ui/window_tracking/macos/mod.rs | 43 +++---- .../src/ui/window_tracking/macos/sys.rs | 6 +- .../src/ui/window_tracking/macos/window.rs | 116 +++++++++++++----- 5 files changed, 129 insertions(+), 62 deletions(-) diff --git a/rust/client/build.rs b/rust/client/build.rs index 0573399..43f5aa5 100644 --- a/rust/client/build.rs +++ b/rust/client/build.rs @@ -13,7 +13,7 @@ use gauntlet_component_model::PropertyType; use gauntlet_component_model::create_component_model; fn main() -> anyhow::Result<()> { - #[cfg(target_os = "macos")] // needed for window focus stuff, specifically SkyLight framework + #[cfg(target_os = "macos")] println!( "cargo:rustc-link-search=framework={}", "/System/Library/PrivateFrameworks" diff --git a/rust/client/src/ui/window_tracking/macos/apps.rs b/rust/client/src/ui/window_tracking/macos/apps.rs index c42eb63..2193b8a 100644 --- a/rust/client/src/ui/window_tracking/macos/apps.rs +++ b/rust/client/src/ui/window_tracking/macos/apps.rs @@ -7,7 +7,12 @@ use accessibility::AXUIElement; use accessibility_sys::kAXRaiseAction; use accessibility_sys::pid_t; use anyhow::Context; +use core_foundation::base::CFType; +use core_foundation::base::FromVoid; +use core_foundation::dictionary::CFDictionary; use core_foundation::string::CFString; +use core_graphics::display::CGDisplay; +use core_graphics::window::kCGWindowListExcludeDesktopElements; use gauntlet_server::plugins::ApplicationManager; use objc2::AnyThread; use objc2::DefinedClass; @@ -24,7 +29,8 @@ use objc2_app_kit::NSWorkspaceDidLaunchApplicationNotification; use objc2_app_kit::NSWorkspaceDidTerminateApplicationNotification; use objc2_foundation::NSNotification; -use super::window::WindowNotificationDelegate; +use super::window::{WindowNotificationDelegate, WindowType}; +use crate::ui::window_tracking::macos::sys::ax_window_id; use crate::ui::window_tracking::macos::sys::make_key_window; pub struct MacosWindowTracker { @@ -84,7 +90,6 @@ impl ApplicationNotificationDelegate { let state = ApplicationNotificationDelegateState { application_manager, applications: RefCell::new(HashMap::new()), - windows: Rc::new(RefCell::new(vec![])), }; let delegate = ApplicationNotificationDelegate::alloc().set_ivars(state); @@ -117,9 +122,8 @@ impl ApplicationNotificationDelegate { fn create_window_notification_delegate(&self, pid: pid_t) -> anyhow::Result<()> { let application_manager = self.ivars().application_manager.clone(); - let windows = self.ivars().windows.clone(); - let delegate = WindowNotificationDelegate::new(pid, application_manager, windows) + let delegate = WindowNotificationDelegate::new(pid, application_manager) .context("Error creating window notification delegate")?; delegate @@ -153,7 +157,16 @@ impl ApplicationNotificationDelegate { println!("Focusing window: {}, {:?}, {}", window_uuid, window, pid); - make_key_window(*pid, window).context("Failed to make window key")?; + if let Some(windows) = CGDisplay::window_list_info(kCGWindowListExcludeDesktopElements, None) { + for item in windows.into_iter() { + let item: CFDictionary = unsafe { CFDictionary::from_void(item.clone()) }.clone(); + println!("CFDictionary: {:?}", item); + } + }; + + let window_id = ax_window_id(window).context("Failed to get window id")?; + + make_key_window(*pid, window_id).context("Failed to make window key")?; // some apps seem to also require additional raise action window @@ -167,7 +180,6 @@ impl ApplicationNotificationDelegate { struct ApplicationNotificationDelegateState { application_manager: Arc, applications: RefCell>, - windows: Rc>>, } define_class!( diff --git a/rust/client/src/ui/window_tracking/macos/mod.rs b/rust/client/src/ui/window_tracking/macos/mod.rs index 3909c17..8e4ba3f 100644 --- a/rust/client/src/ui/window_tracking/macos/mod.rs +++ b/rust/client/src/ui/window_tracking/macos/mod.rs @@ -21,35 +21,29 @@ pub fn request_macos_accessibility_permissions() -> bool { unsafe { AXIsProcessTrustedWithOptions(options.as_concrete_TypeRef()) } } -// todo -// on each ax notification + non-ax destroy event -// get app pid using AXUIElementGetPid -// run bruteforce search (+ regular) for axuielement using _AXUIElementCreateWithRemoteToken -// from each found window axuielement -// filter based on window role/subrole -// get window using _AXUIElementGetWindow -// get title -// to focus use window id and private api +// only active space(s) and fullscreen apps (only single item per app will be shown) are supported +// tabs are supported only on active "desktop" space, not supported in fullscreen window +// show warning when: +// there are non-active spaces +// except fullscreen windows +// there multiple fullscreen windows for specific app +// warning should say: +// gauntlet doesn't show windows on non-active spaces except fullscreen applications +// gauntlet doesn't support fullscreen applications on multiple spaces +// gauntlet doesn't support native tabs for fullscreen applications -// do not support hidden windows -// do not support multiple "desktop" spaces unless they are all visible -// what about multiple fullscreen windows of the same app? -// what about tabs in visible apps in non-visible spaces? -// i.e., only non-hidden windows in visible spaces and maybe(?) fullscreen apps -// support minimized windows but only on visible spaces +// refresh window list when space switches +// if current space is fullscreen do not scan for tabs? show warning? -// support fullscreen applications? -// support tabs on the visible windows in visible spaces only +// CGSSpaceGetType to get type of given space +// CGSGetWindowWorkspace to get list of spaces for specific window +// ? to get space for given +// ? to get current space -// I think ignoring existence of spaces is fine??? +// todo what if there are 2 monitors. is it same space or multiple? what does "separate spaces" setting do? +// todo what if gauntlet started on fullscreen space? -// is the private function for focusing a window needed? - -// todo support for windows on separate spaces -// todo multiple desktop spaces ("Desktop" vs. "Desktop 1" and "Desktop 2") -// todo sometimes the window state seems to be lost, clearing the list of windows -// todo when starting the gauntlet, tabbed windows only show single one // todo implement this https://github.com/glide-wm/glide/issues/10 // todo how to handle system apps and settings wrt window tracking? // todo add all github issue links and appreciations to the commit message @@ -63,5 +57,6 @@ pub fn request_macos_accessibility_permissions() -> bool { // https://github.com/koekeishiya/yabai/issues/68 // https://github.com/koekeishiya/yabai/issues/199#issuecomment-519152388 // https://github.com/lwouis/alt-tab-macos/issues/1324#issuecomment-2631035482 +// https://github.com/glide-wm/glide/issues/10 diff --git a/rust/client/src/ui/window_tracking/macos/sys.rs b/rust/client/src/ui/window_tracking/macos/sys.rs index 99b4fcb..22a3a67 100644 --- a/rust/client/src/ui/window_tracking/macos/sys.rs +++ b/rust/client/src/ui/window_tracking/macos/sys.rs @@ -57,12 +57,10 @@ impl ProcessSerialNumber { } } -pub fn make_key_window(pid: pid_t, window: &AXUIElement) -> anyhow::Result<()> { +pub fn make_key_window(pid: pid_t, window_id: CGWindowID) -> anyhow::Result<()> { // See https://github.com/Hammerspoon/hammerspoon/issues/370#issuecomment-545545468. // god bless all the wizards in that thread that worked on it, thank you - let window_id = ax_window_id(window)?; - println!("window id: {}", window_id); #[allow(non_upper_case_globals)] @@ -95,7 +93,9 @@ pub fn make_key_window(pid: pid_t, window: &AXUIElement) -> anyhow::Result<()> { Ok(()) } +#[allow(unused)] pub fn bruteforce_windows_for_app(app_pid: pid_t) -> Vec { + // this whole thing can take more than a second, do not run on the main thread unsafe { let mut result = vec![]; let mut data = [0; 0x14]; diff --git a/rust/client/src/ui/window_tracking/macos/window.rs b/rust/client/src/ui/window_tracking/macos/window.rs index 31e2017..26d36b9 100644 --- a/rust/client/src/ui/window_tracking/macos/window.rs +++ b/rust/client/src/ui/window_tracking/macos/window.rs @@ -1,9 +1,11 @@ use std::cell::RefCell; +use std::collections::HashMap; use std::ffi::c_void; use std::mem::MaybeUninit; use std::rc::Rc; use std::sync::Arc; +use accessibility::AXAttribute; use accessibility::AXUIElement; use accessibility::AXUIElementAttributes; use accessibility::Error; @@ -15,12 +17,17 @@ use accessibility_sys::AXUIElementRef; use accessibility_sys::kAXDialogSubrole; use accessibility_sys::kAXErrorSuccess; use accessibility_sys::kAXStandardWindowSubrole; +use accessibility_sys::kAXTabGroupRole; +use accessibility_sys::kAXTabsAttribute; use accessibility_sys::kAXTitleChangedNotification; use accessibility_sys::kAXUIElementDestroyedNotification; use accessibility_sys::kAXWindowCreatedNotification; use accessibility_sys::kAXWindowRole; use accessibility_sys::pid_t; use anyhow::Context; +use core_foundation::array::CFArray; +use core_foundation::base::CFType; +use core_foundation::base::FromVoid; use core_foundation::base::TCFType; use core_foundation::runloop::CFRunLoop; use core_foundation::runloop::kCFRunLoopDefaultMode; @@ -32,7 +39,7 @@ use objc2_app_kit::NSRunningApplication; use uuid::Uuid; use super::sys::AXObserver; -use super::sys::bruteforce_windows_for_app; +use super::sys::ax_window_id; pub struct WindowNotificationDelegate { app_element: AXUIElement, @@ -40,9 +47,20 @@ pub struct WindowNotificationDelegate { inner: Rc, } +pub enum WindowType { + Window, + Tab, +} + +struct WindowData { + app_pid: pid_t, + window_type: WindowType, + element: AXUIElement, +} + struct WindowNotificationDelegateInner { app_pid: pid_t, - windows: Rc>>, + windows: Rc>>, application_manager: Arc, } @@ -52,14 +70,8 @@ const WINDOW_EVENTS: [&str; 3] = [ kAXTitleChangedNotification, ]; -const MESSAGING_TIMEOUT_SEC: f32 = 1.0; - impl WindowNotificationDelegate { - pub fn new( - pid: pid_t, - application_manager: Arc, - windows: Rc>>, - ) -> anyhow::Result { + pub fn new(pid: pid_t, application_manager: Arc) -> anyhow::Result { let observer = unsafe { let mut result = MaybeUninit::uninit(); @@ -76,7 +88,7 @@ impl WindowNotificationDelegate { let element = AXUIElement::application(pid); element - .set_messaging_timeout(MESSAGING_TIMEOUT_SEC) + .set_messaging_timeout(1.0) .context("Failed to set messaging timeout")?; element @@ -87,7 +99,7 @@ impl WindowNotificationDelegate { observer, inner: Rc::new(WindowNotificationDelegateInner { app_pid: pid, - windows, + windows: Rc::new(RefCell::new(HashMap::new())), application_manager, }), }) @@ -142,7 +154,7 @@ impl WindowNotificationDelegate { let windows = self.inner.windows.borrow(); - for (window_id, _, _) in windows.iter() { + for (window_id, _) in windows.iter() { let event = MacosWindowTrackingEvent::WindowClosed { window_id: window_id.clone(), }; @@ -205,40 +217,51 @@ fn get_bundle_path(pid: pid_t) -> Option { impl WindowNotificationDelegateInner { fn refresh_windows(&self) -> anyhow::Result<()> { tracing::debug!("Refreshing windows for app: {}", self.app_pid); + let mut retrieved_windows: Vec<_> = AXUIElement::application(self.app_pid) .windows()? .into_iter() - .map(|item| item.clone()) + .map(|window| window.clone()) + .flat_map(|window| { + let tabs = list_tabs(window.clone()); + + if !tabs.is_empty() { + return tabs.into_iter().map(|tab| (WindowType::Tab, tab)).collect::>(); + } + + return vec![(WindowType::Window, window)]; + }) .collect(); - for window in bruteforce_windows_for_app(self.app_pid) { - if !retrieved_windows.contains(&window) { - retrieved_windows.push(window); - }; - } - - tracing::debug!("Retrieved {} windows", retrieved_windows.len()); + tracing::debug!("Retrieved windows: {}", retrieved_windows.len()); let stored_windows = self .windows .borrow() .iter() - .map(|(_, _, window)| window.clone()) + .map(|(_, window)| (window.app_pid.clone(), window.clone())) .collect::>(); - tracing::debug!("Stored {} windows", stored_windows.len()); + tracing::debug!("Stored windows: {}", stored_windows.len()); - for window in stored_windows.into_iter() { - let Some(index) = retrieved_windows.iter().position(|el| el == &window) else { + let mut destroyed_windows = 0; + for (pid, window) in stored_windows.iter() { + if pid != &self.app_pid { + continue; + } + + let Some(index) = retrieved_windows.iter().position(|el| el == window) else { // doesn't exist anymore, destroy it - self.window_destroyed(window); + self.window_destroyed(window.clone()); continue; }; + destroyed_windows += 1; retrieved_windows.swap_remove(index); } - tracing::debug!("left retrieved {} windows", retrieved_windows.len()); + tracing::debug!("Destroyed windows: {}", retrieved_windows.len()); + tracing::debug!("New windows: {}", retrieved_windows.len()); for window in retrieved_windows.into_iter() { self.window_opened(window); @@ -264,7 +287,7 @@ impl WindowNotificationDelegateInner { } let window_id = Uuid::new_v4().to_string(); - windows.push((window_id.clone(), self.app_pid, window.clone())); + windows.insert(window_id.clone(), (self.app_pid, window.clone())); let title = window.title().map(|title| title.to_string()).ok(); let bundle_path = get_bundle_path(self.app_pid); @@ -285,7 +308,7 @@ impl WindowNotificationDelegateInner { return; }; - let (window_id, _, _) = windows.swap_remove(index); + let (window_id, _, _) = windows.remove(index); let event = MacosWindowTrackingEvent::WindowClosed { window_id }; @@ -309,3 +332,40 @@ impl WindowNotificationDelegateInner { self.application_manager.send_macos_window_tracking_event(event); } } + +fn list_tabs(window: AXUIElement) -> Vec { + let Ok(children) = window.children() else { + return vec![]; + }; + + let tab_group = children.into_iter().find(|child| { + let role = child.role().map(|val| val.to_string()).ok(); + if role.as_deref() != Some(kAXTabGroupRole) { + return false; + } + + let title = child.title().map(|val| val.to_string()).ok(); + if title.as_deref() != Some("tab bar") { + return false; + } + + return true; + }); + + let Some(tab_group) = tab_group else { + return vec![]; + }; + + let tabs_attribute = AXAttribute::::new(&CFString::from_static_string(kAXTabsAttribute)); + let Some(tabs) = tab_group.attribute(&tabs_attribute).ok() else { + return vec![]; + }; + + let Some(tabs) = tabs.downcast::() else { + return vec![]; + }; + + tabs.into_iter() + .map(|item| unsafe { AXUIElement::from_void(item.clone()) }.clone()) + .collect() +}