Why does a custom Window Procedure stop processing window messages?

Question

I'm trying to configure a custom window procedure and it works. However, after a while, the window stops reacting to any input. It seems that the more rendering is going on in a scene, the sooner the window gets broken.

This even happens if my custom window procedure simply calls the default window.

Reproducer:

package com.example;

import com.sun.jna.Native;
import com.sun.jna.Pointer;
import com.sun.jna.platform.win32.BaseTSD;
import com.sun.jna.platform.win32.User32;
import com.sun.jna.platform.win32.WinDef.HWND;
import com.sun.jna.platform.win32.WinUser;
import com.sun.jna.win32.W32APIOptions;
import javafx.application.Application;
import javafx.scene.Scene;
import javafx.scene.control.ProgressIndicator;
import javafx.stage.Stage;

import static com.sun.jna.platform.win32.WinUser.GWL_WNDPROC;

public class CustomWndProc {

  public static void main(String[] args) {
    CustomFrameApplication.launch(CustomFrameApplication.class, args);
  }

  public static class CustomFrameApplication extends Application {

    @Override
    public void start(Stage primaryStage) {
      primaryStage.setScene(new Scene(new ProgressIndicator(ProgressIndicator.INDETERMINATE_PROGRESS)));
      primaryStage.show();

      HWND hwnd = new HWND();
      hwnd.setPointer(User32.INSTANCE.GetActiveWindow().getPointer());

      BaseTSD.LONG_PTR defaultWindowProc = User32.INSTANCE.GetWindowLongPtr(hwnd, GWL_WNDPROC);

      WinUser.WindowProc windowProc = (hwnd1, uMsg, wParam, lParam) ->
        User32Ex.INSTANCE.CallWindowProc(defaultWindowProc, hwnd1, uMsg, wParam, lParam);

      User32Ex.INSTANCE.SetWindowLongPtr(hwnd, GWL_WNDPROC, windowProc);
    }
  }

  public interface User32Ex extends User32 {
    User32Ex INSTANCE = Native.load("user32", User32Ex.class, W32APIOptions.DEFAULT_OPTIONS);

    Pointer SetWindowLongPtr(HWND hWnd, int nIndex, WindowProc wndProc);

    LRESULT CallWindowProc(LONG_PTR proc, HWND hWnd, int uMsg, WPARAM uParam, LPARAM lParam);
  }
}

Give it a few seconds or minutes and you won't be able to move, minimize, maximize or close the window anymore.

If you want guaranteed freeze, use a WebView instead of a ProgressIndicator:

      WebView webView = new WebView();
      webView.getEngine().load("https://www.google.com");
      primaryStage.setScene(new Scene(webView));

I wondered if it has something to do that my code runs in the JavaFX Application thread (leading to some race condition) but I assume so does the default window procedure (how can I verify?).

I'm trying to build a JavaFX application that uses a custom frame.

Using JNA 5.5.0.

Answer 1

I think this is the right way to do it, but maybe I'm wrong.. I got the Information from: Why my JNA using application doesn't react in a right way?

import com.sun.jna.LastErrorException;
import com.sun.jna.Native;
import com.sun.jna.platform.win32.BaseTSD;
import com.sun.jna.platform.win32.User32;
import com.sun.jna.platform.win32.WinDef;
import com.sun.jna.platform.win32.WinDef.HWND;
import com.sun.jna.win32.W32APIOptions;
import javafx.application.Application;
import javafx.scene.Scene;
import javafx.scene.control.ProgressIndicator;
import javafx.stage.Stage;


public class CustomWndProc {

    public static void main(String[] args) {
        CustomFrameApplication.launch(CustomFrameApplication.class, args);
    }

    public static class CustomFrameApplication extends Application {

        private BaseTSD.LONG_PTR baseWndProc;

        public User32Ex.WNDPROC listener = new User32Ex.WNDPROC() {
            public WinDef.LRESULT callback(HWND hWnd, int uMsg, WinDef.WPARAM uParam,
                    WinDef.LPARAM lParam) {
                // TODO handle the window message
                // calling the base WndProc
                return User32Ex.INSTANCE.CallWindowProc(baseWndProc, hWnd, uMsg, uParam, lParam);
            }
        };

        @Override
        public void start(Stage primaryStage) {
            primaryStage.setScene(new Scene(new ProgressIndicator(ProgressIndicator.INDETERMINATE_PROGRESS)));
            primaryStage.show();

            HWND hwnd = new HWND();
            hwnd.setPointer(User32.INSTANCE.GetActiveWindow().getPointer());

            this.baseWndProc = User32Ex.INSTANCE.GetWindowLongPtr(hwnd, User32Ex.GWL_WNDPROC);

            User32Ex.INSTANCE.SetWindowLongPtr(hwnd, User32Ex.GWL_WNDPROC, this.listener);
        }
    }

    public interface User32Ex extends User32 {

        User32Ex INSTANCE = Native.load("user32", User32Ex.class, W32APIOptions.DEFAULT_OPTIONS);

        interface WNDPROC extends StdCallCallback {

            LRESULT callback(HWND hWnd, int uMsg, WPARAM uParam, LPARAM lParam);
        }

        LONG_PTR GetWindowLongPtr(HWND hWnd, int nIndex) throws LastErrorException;

        LRESULT CallWindowProc(LONG_PTR proc, HWND hWnd, int uMsg, WPARAM uParam, WinDef.LPARAM lParam) throws LastErrorException;

        LONG_PTR SetWindowLongPtr(HWND hWnd, int nIndex, WNDPROC wndProc) throws LastErrorException;
    }
}