How do I get F12 to break into the debugger on Vista x64 with VS 2008?

Question

F12 is a wonder for tracking down UI blocking operations, but I can't figure out how to make it work with VS 2008 and managed code.

Help! Or not...

Edit: turns out it doesn't work in VS 2005 either on Vista x64, so I guess that either widens or narrows it down, depending on your perspective :(

MSN

Answer 1

This is an alternative solution that preserves the convenience of hitting F12 at any time, even if the application "bites", without having to switch to Visual Studio and invoke "Break all" command as per my first answer.

Unfortunately, this solution requires one to add some extra code to the application we want to break into using F12. This special debugging code could be conditionally-compiled using e.g. DEBUG symbol so that the F12 functionality is not available in release builds.

The solution works by creating a background thread; the thread then registers a global keyboard hook and starts a form-less message loop. The sample application below creates a form on the main UI thread with a single button, "Sleep". When the button in clicked, main UI thread will be locked by sleeping for 10 seconds (Thread.Sleep) to simulate application "biting".

To test the F12 functionality, first click "Sleep" button, then hit F12 - program will break into debugger immediately even though the main UI thread is being blocked. The following screenshot shows currently running threads on my machine (Vista x64) after F12:

Active threads on F12 http://img242.imageshack.us/img242/5189/f12threadsre9.png

As you can see, the main program is still in the "GoToSleep" method, while the global hook is invoked in the background thread we created.

The global hook code is based on Stephen Toub's article.

Now the implementation:

using System;
using System.Diagnostics;
using System.Drawing;
using System.Runtime.InteropServices;
using System.Threading;
using System.Windows.Forms;

namespace KeyboardHook
{

    public sealed class SimpleKeyboardHook : IDisposable
    {
        public SimpleKeyboardHook(Action<Keys> handler)
        {
            if (null == handler) { throw new ArgumentNullException("handler"); }
            this._handler = handler;
            var t = new Thread(this.ListenerThread) { IsBackground = true, Name = "KeyboardListener" };
            t.Start();
        }

        public void Dispose()
        {
            if (!this._disposed)
            {
                UnhookWindowsHookEx(this._id);
                this._disposed = true;
                GC.SuppressFinalize(this);
            }
        }

        public static void BreakOnF12(Keys keys)
        {
            if (keys == Keys.F12)
            {
                Debugger.Break();
            }
        }

        private void ListenerThread()
        {
            using (var currentProcess = Process.GetCurrentProcess())
            using (var mainModule = currentProcess.MainModule)
            {
                if (null == mainModule) { throw new InvalidOperationException("Unable to determine main module for the current process"); }

                this._id = SetWindowsHookEx(
                    WH_KEYBOARD_LL,
                    this.HookCallback,
                    GetModuleHandle(mainModule.ModuleName), 0);
            }

            Application.Run();
        }

        private IntPtr HookCallback(int nCode, IntPtr wParam, IntPtr lParam)
        {
            if (nCode >= 0 && wParam == (IntPtr)WM_KEYDOWN)
            {
                var vkCode = Marshal.ReadInt32(lParam);
                this._handler((Keys)vkCode);
            }
            return CallNextHookEx(this._id, nCode, wParam, lParam);
        }

        private delegate IntPtr LowLevelKeyboardProc(int nCode, IntPtr wParam, IntPtr lParam);

        [DllImport("user32.dll", CharSet = CharSet.Auto, SetLastError = true)]
        private static extern IntPtr SetWindowsHookEx(int idHook,
            LowLevelKeyboardProc lpfn, IntPtr hMod, uint dwThreadId);

        [DllImport("user32.dll", CharSet = CharSet.Auto, SetLastError = true)]
        [return: MarshalAs(UnmanagedType.Bool)]
        private static extern bool UnhookWindowsHookEx(IntPtr hhk);

        [DllImport("user32.dll", CharSet = CharSet.Auto, SetLastError = true)]
        private static extern IntPtr CallNextHookEx(IntPtr hhk, int nCode,
            IntPtr wParam, IntPtr lParam);

        [DllImport("kernel32.dll", CharSet = CharSet.Auto, SetLastError = true)]
        private static extern IntPtr GetModuleHandle(string lpModuleName);

        private const int WH_KEYBOARD_LL = 13;
        private const int WM_KEYDOWN = 0x0100;
        private IntPtr _id;
        private readonly Action<Keys> _handler;
        private volatile bool _disposed;
    }


    static class Program
    {
        private static void GoToSleep(object sender, EventArgs args)
        {
            Thread.Sleep(10000);
        }

        [STAThread]
        static void Main()
        {
            using (new SimpleKeyboardHook(SimpleKeyboardHook.BreakOnF12))
            {
                Application.EnableVisualStyles();
                Application.SetCompatibleTextRenderingDefault(false);

                var form = new Form { Text = "Sleepy form", Size = new Size(160,80), Padding = new Padding(6) };
                var btn = new Button { Dock = DockStyle.Fill, Text = "Sleep", Location = new Point(10, 10) };
                btn.Click += GoToSleep;
                form.Controls.Add(btn);
                Application.Run(form);
            }
        }
    }
}