How to get global screen coordinates of currently selected text via Accessibility APIs.

Question

I need help to find out, how Dictionary app showing following popup dialog for selected text on pressing CMD+CTRL+D on any application. I want to implement the same kind of functionality for my cocoa app, where my app will run in background and showing suggestions on some hot key press for the selected text.

I have already implemented hot key capturing, i just need to have code to get the rectangle area of selected text on screen, so i can show the dialog like dictionary app.

Thanks

Answer 1

You can use the accessibility APIs for that. Make sure that the "Enable access for assistive devices" setting is checked (in System Preferences / Universal Access).

The following code snippet will determine the bounds (in screen coordinates) of the selected text in most applications. Unfortunately, it doesn't work in Mail and Safari, because they use private accessibility attributes. It's probably possible to get it to work there as well, but it requires more work and possibly private API calls.

AXUIElementRef systemWideElement = AXUIElementCreateSystemWide();
AXUIElementRef focussedElement = NULL;
AXError error = AXUIElementCopyAttributeValue(systemWideElement, kAXFocusedUIElementAttribute, (CFTypeRef *)&focussedElement);
if (error != kAXErrorSuccess) {
    NSLog(@"Could not get focussed element");
} else {
    AXValueRef selectedRangeValue = NULL;
    AXError getSelectedRangeError = AXUIElementCopyAttributeValue(focussedElement, kAXSelectedTextRangeAttribute, (CFTypeRef *)&selectedRangeValue);
    if (getSelectedRangeError == kAXErrorSuccess) {
        CFRange selectedRange;
        AXValueGetValue(selectedRangeValue, kAXValueCFRangeType, &selectedRange);
        AXValueRef selectionBoundsValue = NULL;
        AXError getSelectionBoundsError = AXUIElementCopyParameterizedAttributeValue(focussedElement, kAXBoundsForRangeParameterizedAttribute, selectedRangeValue, (CFTypeRef *)&selectionBoundsValue);
        CFRelease(selectedRangeValue);
        if (getSelectionBoundsError == kAXErrorSuccess) {
            CGRect selectionBounds;
            AXValueGetValue(selectionBoundsValue, kAXValueCGRectType, &selectionBounds);
            NSLog(@"Selection bounds: %@", NSStringFromRect(NSRectFromCGRect(selectionBounds)));
        } else {
            NSLog(@"Could not get bounds for selected range");
        }
        if (selectionBoundsValue != NULL) CFRelease(selectionBoundsValue);
    } else {
        NSLog(@"Could not get selected range");
    }
}
if (focussedElement != NULL) CFRelease(focussedElement);
CFRelease(systemWideElement);

Answer 2

Here you go for @omz answer in Swift

    let systemWideElement = AXUIElementCreateSystemWide()
    var focusedElement : AnyObject?

    let error = AXUIElementCopyAttributeValue(systemWideElement, kAXFocusedUIElementAttribute as CFString, &focusedElement)
    if (error != .success){
        print("Couldn't get the focused element. Probably a webkit application")
    } else {
        var selectedRangeValue : AnyObject?
        let selectedRangeError = AXUIElementCopyAttributeValue(focusedElement as! AXUIElement, kAXSelectedTextRangeAttribute as CFString, &selectedRangeValue)
        if (selectedRangeError == .success){
            var selectedRange : CFRange?
            AXValueGetValue(selectedRangeValue as! AXValue, AXValueType(rawValue: kAXValueCFRangeType)!, &selectedRange)
            var selectRect = CGRect()
            var selectBounds : AnyObject?
            let selectedBoundsError = AXUIElementCopyParameterizedAttributeValue(focusedElement as! AXUIElement, kAXBoundsForRangeParameterizedAttribute as CFString, selectedRangeValue!, &selectBounds)
            if (selectedBoundsError == .success){
                AXValueGetValue(selectBounds as! AXValue, .cgRect, &selectRect)
                //do whatever you want with your selectRect
                print(selectRect)
            }
        }
    }