pymouse.click not interfacing with other software

Question

I have used pymouse to help automate repetitive games in the past with success.

However, when playing a game downloaded on BlueStacks, pymouse will appear to move to the correct position on the screen, but then no clicks will "register".

If I put something else native to the OS in the same position, it will be clicked. I do not understand why the clicks dont "work" in this case when I move the mouse to a position over the game being played in Bluestacks.

Here is the code:

from pymouse import PyMouse
import time

m = PyMouse()

i=1
for i in range(1,1000):
        time.sleep(2)
        x, y = m.position()
        print(x, y)
        m.click(x,y,1)
        i+=1

This (below) will return values even if the mouse is hovered over the window from Bluestacks.

print(m.position())

Answer 1

There is one option left open to you without resorting to a virtual machine or writing a custom driver. if you install a hook for mouse commands you have the capability to alter the flags in that hook before passing them to the next hook.

Due to how windows queues the hooks you need the hook altering the flags to be the last registered hook, so in this example you would need to start your game before installing the hook.

some example code to do this is below, I tested this using pyHook to check the result. for your purposes you will probably want to wrap this in a thread so it happens in the background.

import atexit
import ctypes
import time
from ctypes import c_short, c_char, c_uint8, c_int32, c_int, c_uint, c_uint32, c_long, byref, Structure, CFUNCTYPE, POINTER
from ctypes.wintypes import DWORD, BOOL, HHOOK, MSG, LPWSTR, WCHAR, WPARAM, LPARAM
from collections import namedtuple
LPMSG = POINTER(MSG)

user32 = ctypes.WinDLL('user32', use_last_error = True)

class MSLLHOOKSTRUCT(Structure):
    _fields_ = [("x", c_long),
                ("y", c_long),
                ('data', c_int32),
                ("flags", DWORD),
                ("time", c_int),
                ('extrainfo', c_int32),
                ]

LowLevelMouseProc = CFUNCTYPE(c_int, WPARAM, LPARAM, POINTER(MSLLHOOKSTRUCT))

SetWindowsHookEx = user32.SetWindowsHookExA
#SetWindowsHookEx.argtypes = [c_int, LowLevelMouseProc, c_int, c_int]
SetWindowsHookEx.restype = HHOOK

CallNextHookEx = user32.CallNextHookEx
#CallNextHookEx.argtypes = [c_int , c_int, c_int, POINTER(MSLLHOOKSTRUCT)]
CallNextHookEx.restype = c_int

UnhookWindowsHookEx = user32.UnhookWindowsHookEx
UnhookWindowsHookEx.argtypes = [HHOOK]
UnhookWindowsHookEx.restype = BOOL

GetMessage = user32.GetMessageW
GetMessage.argtypes = [LPMSG, c_int, c_int, c_int]
GetMessage.restype = BOOL

TranslateMessage = user32.TranslateMessage
TranslateMessage.argtypes = [LPMSG]
TranslateMessage.restype = BOOL

DispatchMessage = user32.DispatchMessageA
DispatchMessage.argtypes = [LPMSG]

# Beware, as of 2016-01-30 the official docs have a very incomplete list.
# This one was compiled from experience and may be incomplete.
WM_MOUSEMOVE = 0x200
WM_LBUTTONDOWN = 0x201
WM_LBUTTONUP = 0x202
WM_LBUTTONDBLCLK = 0x203
WM_RBUTTONDOWN = 0x204
WM_RBUTTONUP = 0x205
WM_RBUTTONDBLCLK = 0x206
WM_MBUTTONDOWN = 0x207
WM_MBUTTONUP = 0x208
WM_MBUTTONDBLCLK = 0x209
WM_MOUSEWHEEL = 0x20A
WM_XBUTTONDOWN = 0x20B
WM_XBUTTONUP = 0x20C
WM_XBUTTONDBLCLK = 0x20D
WM_NCXBUTTONDOWN = 0x00AB
WM_NCXBUTTONUP = 0x00AC
WM_NCXBUTTONDBLCLK = 0x00AD
WM_MOUSEHWHEEL = 0x20E
WM_LBUTTONDOWN = 0x0201
WM_LBUTTONUP = 0x0202
WM_MOUSEMOVE = 0x0200
WM_MOUSEWHEEL = 0x020A
WM_MOUSEHWHEEL = 0x020E
WM_RBUTTONDOWN = 0x0204
WM_RBUTTONUP = 0x0205

LEFT = 'left'
RIGHT = 'right'
MIDDLE = 'middle'
X = 'x'

UP = 'up'
DOWN = 'down'
DOUBLE = 'double'

buttons_by_wm_code = {
    WM_LBUTTONDOWN: (DOWN, LEFT),
    WM_LBUTTONUP: (UP, LEFT),
    WM_LBUTTONDBLCLK: (DOUBLE, LEFT),

    WM_RBUTTONDOWN: (DOWN, RIGHT),
    WM_RBUTTONUP: (UP, RIGHT),
    WM_RBUTTONDBLCLK: (DOUBLE, RIGHT),

    WM_MBUTTONDOWN: (DOWN, MIDDLE),
    WM_MBUTTONUP: (UP, MIDDLE),
    WM_MBUTTONDBLCLK: (DOUBLE, MIDDLE),

    WM_XBUTTONDOWN: (DOWN, X),
    WM_XBUTTONUP: (UP, X),
    WM_XBUTTONDBLCLK: (DOUBLE, X),
}

NULL = c_int(0)

def translate_injected_mouse():
    def low_level_mouse_handler(nCode, wParam, lParam):
        struct = lParam.contents

        if wParam in buttons_by_wm_code:
            struct.flags &= 0x11111100 # clear the injected flags
        return CallNextHookEx(NULL, nCode, wParam, lParam)

    WH_MOUSE_LL = c_int(14)
    mouse_callback = LowLevelMouseProc(low_level_mouse_handler)
    mouse_hook = SetWindowsHookEx(WH_MOUSE_LL, mouse_callback, user32._handle, NULL)

    # Register to remove the hook when the interpreter exits. Unfortunately a
    # try/finally block doesn't seem to work here.
    atexit.register(UnhookWindowsHookEx, mouse_hook)

    msg = LPMSG()
    while not GetMessage(msg, NULL, NULL, NULL):
        TranslateMessage(msg)
        DispatchMessage(msg)

if __name__ == '__main__':
    translate_injected_mouse()

stripped down example that wraps functionality into a thread class:

import atexit
import ctypes
from ctypes import c_int, c_uint, c_uint32, c_long, Structure, CFUNCTYPE, POINTER
from ctypes.wintypes import DWORD, BOOL, HWND, HHOOK, MSG, WPARAM, LPARAM
import threading

LPMSG = POINTER(MSG)

user32 = ctypes.WinDLL('user32', use_last_error = True)

class MSLLHOOKSTRUCT(Structure):
    _fields_ = [("x", c_long), ("y", c_long),
                ('data', c_uint32), ("flags", DWORD),
                ("time", c_int), ('extrainfo', c_uint32), ]

LowLevelMouseProc = CFUNCTYPE(c_int, WPARAM, LPARAM, POINTER(MSLLHOOKSTRUCT))

SetWindowsHookEx = user32.SetWindowsHookExA
#SetWindowsHookEx.argtypes = [c_int, LowLevelMouseProc, c_int, c_int]
SetWindowsHookEx.restype = HHOOK

CallNextHookEx = user32.CallNextHookEx
#CallNextHookEx.argtypes = [c_int , c_int, c_int, POINTER(MSLLHOOKSTRUCT)]
CallNextHookEx.restype = c_int

UnhookWindowsHookEx = user32.UnhookWindowsHookEx
UnhookWindowsHookEx.argtypes = [HHOOK]
UnhookWindowsHookEx.restype = BOOL

GetMessage = user32.GetMessageW
GetMessage.argtypes = [LPMSG, c_int, c_int, c_int]
GetMessage.restype = BOOL

TranslateMessage = user32.TranslateMessage
TranslateMessage.argtypes = [LPMSG]
TranslateMessage.restype = BOOL

DispatchMessage = user32.DispatchMessageA
DispatchMessage.argtypes = [LPMSG]

NULL = c_int(0)

class TranslateInjectedMouse(threading.Thread):
    daemon=True
    def run(self):
        def low_level_mouse_handler(nCode, wParam, lParam):
            print("handler")
            lParam.contents.flags &= 0x11111100
            return CallNextHookEx(NULL, nCode, wParam, lParam)

        WH_MOUSE_LL = c_int(14)
        mouse_callback = LowLevelMouseProc(low_level_mouse_handler)
        self.mouse_hook = SetWindowsHookEx(WH_MOUSE_LL, mouse_callback, user32._handle, NULL)

        # Register to remove the hook when the interpreter exits. Unfortunately a
        # try/finally block doesn't seem to work here.
        atexit.register(UnhookWindowsHookEx, self.mouse_hook)

        msg = LPMSG()
        while not GetMessage(msg, NULL, NULL, NULL):
            TranslateMessage(msg)
            DispatchMessage(msg)

    def stop(self):
        UnhookWindowsHookEx(self.mouse_hook)

if __name__ == '__main__':
    # this is all you need to translate in background
    t = TranslateInjectedMouse()
    t.start()

    # below this is test code to create clicks
    import time
    mouse_event = user32.mouse_event
    MOUSEEVENTF_LEFTDOWN = 0x0002
    MOUSEEVENTF_LEFTUP = 0x0004

    while True:
        try:
            time.sleep(1)
            mouse_event(MOUSEEVENTF_LEFTDOWN, 0, 0, 0, 0)

            mouse_event(MOUSEEVENTF_LEFTUP, 0, 0, 0, 0)
        except KeyboardInterrupt:
            if t.is_alive():
                t.stop()
            else:
                break

note that presently I cannot find a way to kill this thread on command. the thread execution blocks at GetMessage waiting for messages to handle such as WM_QUIT, because this happens in C rather than in python raising an exception in the thread does not cause it to exit, I've tried a large number of combinations such as SendMessage, PostMessage and PostThreadMessage to try and send a WM_QUIT message but with no success. instead the daemon flag is set which forces it to exit when the main thread exits.

Answer 2

It use mouse_event which called SendInput internally.

The SendInput function will insert input events into the same queue as a hardware device but the events are marked with a LLMHF_INJECTED flag that can be detected by hooks. To avoid this flag you probably have to write a custom driver.

For your special case,i think you can use SetWindowsHookEx clear the flag. but for easy use,just using a VMware.please see this post.

Answer 3

try this, it work for me.

# -*- coding: UTF-8 -*-

import pyautogui as gui
from time import sleep
from threading import Thread

from pymouse import PyMouse


__author__ = 'lpe234'


def click(x, y):
    print(x, y)
    gui.click(x, y)


def main():
    # make sure the window is active. pyautogui.click can't active the window, don't know why
    PyMouse().click(489, 316)

    t = Thread(target=click, args=[400, 500])
    t.daemon = True
    t.start()
    sleep(5)


if __name__ == '__main__':
    main()