JNA Keyboard Hook in Windows

Question

I have put together a JNA code for installing keyboard hook in Windows (using the JNA examples). The code compiles and everything, and I get the hook installed (I get handle to the hook successfully), also I can uninstall the hook successfully. However, the callback never get called when I press any key on the keyboard. Here is my code (most of it are type definitions got from the JNA examples, go to the "main" directly for my part)

import com.sun.jna.IntegerType;
import com.sun.jna.Pointer;
import com.sun.jna.PointerType;
import com.sun.jna.Structure;
import com.sun.jna.FromNativeContext;
import com.sun.jna.ptr.IntByReference;
import com.sun.jna.ptr.PointerByReference;
import com.sun.jna.win32.StdCallLibrary;
import com.sun.jna.win32.StdCallLibrary.StdCallCallback;
import com.sun.jna.Native;
import com.sun.jna.Platform;
import com.sun.jna.Library;
import com.sun.jna.win32.W32APITypeMapper;
import com.sun.jna.win32.W32APIFunctionMapper;

import java.util.Map;
import java.util.HashMap;

public class HelloWorld {
    static Map UNICODE_OPTIONS = new HashMap() {
        {
            put("type-mapper", W32APITypeMapper.UNICODE);
            put("function-mapper", W32APIFunctionMapper.UNICODE);
        }
    };

    public static class LONG_PTR extends IntegerType {
        public LONG_PTR() { this(0); }
        public LONG_PTR(long value) { super(Pointer.SIZE, value); }
    }

    public static class UINT_PTR extends IntegerType {
        public UINT_PTR() { super(Pointer.SIZE); }
        public UINT_PTR(long value) { super(Pointer.SIZE, value); }
        public Pointer toPointer() { return Pointer.createConstant(longValue()); }
    }

    public static class ULONG_PTR extends IntegerType {
        public ULONG_PTR() { this(0); }
        public ULONG_PTR(long value) { super(Pointer.SIZE, value); }
    }

    public static class LRESULT extends LONG_PTR {
        public LRESULT() { this(0); }
        public LRESULT(long value) { super(value); }
    }

    public static class WPARAM extends UINT_PTR {
        public WPARAM() { this(0); }
        public WPARAM(long value) { super(value); }
    }

    public static class LPARAM extends LONG_PTR {
        public LPARAM() { this(0); }
        public LPARAM(long value) { super(value); }
    }

    public static class KBDLLHOOKSTRUCT extends Structure {
        public int vkCode;
        public int scanCode;
        public int flags;
        public int time;
        public ULONG_PTR dwExtraInfo;
    }

    static HANDLE INVALID_HANDLE_VALUE = new HANDLE() {
        { super.setPointer(Pointer.createConstant(-1)); }
        public void setPointer(Pointer p) {
            throw new UnsupportedOperationException("Immutable reference");
        }
    };

    public static class HANDLE extends PointerType {
        public Object fromNative(Object nativeValue, FromNativeContext context) {
            Object o = super.fromNative(nativeValue, context);
            if (INVALID_HANDLE_VALUE.equals(o))
                return INVALID_HANDLE_VALUE;
            return o;
        }
    }

    public static class HHOOK extends HANDLE { }
    public static class HINSTANCE extends HANDLE { }
    public static class HMODULE extends HINSTANCE { }

    public interface User32 extends StdCallLibrary  {
        User32 INSTANCE = (User32)Native.loadLibrary("user32", User32.class, UNICODE_OPTIONS);

        static final int WH_KEYBOARD_LL = 13;

        public static interface HOOKPROC extends StdCallCallback  {
            LRESULT callback(int nCode, WPARAM wParam, KBDLLHOOKSTRUCT lParam);
        }

        HHOOK SetWindowsHookEx(int idHook, HOOKPROC lpfn, HMODULE hMod, int dwThreadId);
        LRESULT CallNextHookEx(HHOOK idHook, int nCode, WPARAM wParam, LPARAM lParam);
        LRESULT CallNextHookEx(HHOOK idHook, int nCode, WPARAM wParam, Pointer lParam);

        boolean UnhookWindowsHookEx(HHOOK idHook);
    }

    public interface Kernel32 extends StdCallLibrary {
        Kernel32 INSTANCE = (Kernel32)Native.loadLibrary("kernel32", Kernel32.class, UNICODE_OPTIONS);

        HMODULE GetModuleHandle(String name);
    }

    public static HHOOK hHook;
    public static User32.HOOKPROC lpfn;
    public static volatile boolean quit = false;

    public static void main(String[] args) throws Exception {
        HMODULE hMod = Kernel32.INSTANCE.GetModuleHandle(null);
        System.out.println(hMod);

        lpfn = new User32.HOOKPROC() {
            public LRESULT callback(int nCode, WPARAM wParam, KBDLLHOOKSTRUCT lParam) {
                System.out.println("here");
                quit = true;
                return User32.INSTANCE.CallNextHookEx(hHook, nCode, wParam, lParam.getPointer());
            }
        };

        hHook = User32.INSTANCE.SetWindowsHookEx(User32.WH_KEYBOARD_LL, lpfn, hMod, 0);
        System.out.println(hHook);

        if(hHook != null)
            System.out.println("Keyboard hooked, type anything to quit");

        while(!quit) {
            Thread.sleep(100);
        }

        if(User32.INSTANCE.UnhookWindowsHookEx(hHook))
            System.out.println("Unhooked");

    }
}

I have done keyboard/mouse hooks several times using both C++ and C# in the past. This my first attempt with Java, and I just don't know whether I imported and mapped the library correctly. Any ideas?

Thank you.

Answer 1

It appears you need to call GetMessage or PeekMessage, which is odd - it is not mentioned in the documentation for Hooks or LowLevelKeyboardProc. I don't know enough about this part of the API to guess at the reason.

I just used the example classes:

import com.sun.jna.examples.win32.*;

public class Callback {
  public static User32.HHOOK hHook;
  public static User32.LowLevelKeyboardProc lpfn;
  public static volatile boolean quit = false;

  public static void main(String[] args) throws Exception {
    W32API.HMODULE hMod = Kernel32.INSTANCE.GetModuleHandle(null);
    lpfn = new User32.LowLevelKeyboardProc() {
      public W32API.LRESULT callback(int nCode, W32API.WPARAM wParam,
          User32.KBDLLHOOKSTRUCT lParam) {
        System.out.println("here");
        quit = true;
        return User32.INSTANCE.CallNextHookEx(hHook, nCode, wParam, lParam
            .getPointer());
      }
    };
    hHook = User32.INSTANCE.SetWindowsHookEx(User32.WH_KEYBOARD_LL, lpfn, hMod,
        0);
    if (hHook == null)
      return;
    User32.MSG msg = new User32.MSG();
    while (!quit) {
      User32.INSTANCE.PeekMessage(msg, null, 0, 0, 0);
      Thread.sleep(100);
    }
    if (User32.INSTANCE.UnhookWindowsHookEx(hHook))
      System.out.println("Unhooked");
  }
}

Answer 2

A minimalist example:

import com.sun.jna.platform.win32.Kernel32;
import com.sun.jna.platform.win32.User32;
import com.sun.jna.platform.win32.WinDef.HINSTANCE;
import com.sun.jna.platform.win32.WinDef.LPARAM;
import com.sun.jna.platform.win32.WinDef.LRESULT;
import com.sun.jna.platform.win32.WinDef.WPARAM;
import com.sun.jna.platform.win32.WinUser.HOOKPROC;

public class MainTestKeyHook {


    public static void main(String[] args) throws Exception {
        HOOKPROC hookProc = new HOOKPROC_bg();
        HINSTANCE hInst = Kernel32.INSTANCE.GetModuleHandle(null);

        User32.HHOOK hHook = User32.INSTANCE.SetWindowsHookEx(User32.WH_KEYBOARD_LL, hookProc, hInst, 0);
        if (hHook == null)
            return;
        User32.MSG msg = new User32.MSG();
        System.err.println("Please press any key ....");
        while (true) {
            User32.INSTANCE.GetMessage(msg, null, 0, 0);
        }   
    }
}

class HOOKPROC_bg implements HOOKPROC {

    public HOOKPROC_bg() {
    }

    public LRESULT callback(int nCode, WPARAM wParam, LPARAM lParam) {
        System.err.println("callback bbbnhkilhjkibh nCode: " + nCode);
        return new LRESULT(0);
    }
}