IsMouseOver returns false over some elements in a DockPanel

Question

I'm trying to detect when mouse enters VS 2017 title bar, but I've noticed that MouseEnter and MouseLeave events don't work correctly. Event fires only when mouse enters child controls outlined by green rectangle on the screenshot below.

The title bar is a DockPanel with some elements in it. I've set its background to SolidColorBrush(Colors.Red) to make sure hit test runs correctly. When mouse is over elements in green rectangle IsMouseOver correctly returns true, but everywhere else it is false. For menu bar, IsMouseOver and MouseEnter and MouseLeave events work correctly. What could be wrong there?

Update 2: It is likely that title bar is marked as non-client area and this is what causes this problem

Update:

Here is Visual Tree of main VS window:

Decompiled MainWindowTitleBar class:

using Microsoft.VisualStudio.PlatformUI.Shell.Controls;
using Microsoft.VisualStudio.Shell;
using System;
using System.Windows;
using System.Windows.Automation.Peers;
using System.Windows.Controls;
using System.Windows.Input;
using System.Windows.Interop;
using System.Windows.Media;

namespace Microsoft.VisualStudio.PlatformUI
{
    public sealed class MainWindowTitleBar : Border, INonClientArea
    {
        protected override HitTestResult HitTestCore(PointHitTestParameters hitTestParameters)
        {
            return new PointHitTestResult(this, hitTestParameters.HitPoint);
        }

        int INonClientArea.HitTest(Point point)
        {
            return 2;
        }

        protected override AutomationPeer OnCreateAutomationPeer()
        {
            return new MainWindowTitleBarAutomationPeer(this);
        }

        protected override void OnContextMenuOpening(ContextMenuEventArgs e)
        {
            if (!e.Handled)
            {
                HwndSource hwndSource = PresentationSource.FromVisual(this) as HwndSource;
                if (hwndSource != null)
                {
                    CustomChromeWindow.ShowWindowMenu(hwndSource, this, Mouse.GetPosition(this), base.RenderSize);
                }
                e.Handled = true;
            }
        }
    }
}

Extracted XAML for MainWindowTitleBar:

<mwtb:MainWindowTitleBar Name="MainWindowTitleBar" x:Uid="vs:MainWindowTitleBar_1" Grid.Row="0" Grid.Column="0" Background="{DynamicResource {x:Static ui:EnvironmentColors.MainWindowActiveCaptionBrushKey}}" TextElement.Foreground="{DynamicResource {x:Static ui:EnvironmentColors.MainWindowActiveCaptionTextBrushKey}}">
  <DockPanel x:Uid="DockPanel_2">
    <wcp:SystemMenu Name="SystemMenu" x:Uid="Image_1" Source="{TemplateBinding Window.Icon}" VectorFill="{DynamicResource {x:Static ui:EnvironmentColors.MainWindowActiveIconDefaultBrushKey}}" Width="32" Height="27" Margin="0,0,12,4" Padding="12,7,0,0" DockPanel.Dock="Left" VectorIcon="{Binding Source={x:Static Application.Current}, Path=VectorIcon}" />
    <StackPanel Name="WindowTitleBarButtons" x:Uid="WindowTitleBarButtons" Orientation="Horizontal" DockPanel.Dock="Right">
      <wcp:WindowTitleBarButton Name="MinimizeButton" x:Uid="MinimizeButton" VerticalAlignment="Top" Command="{x:Static vsc:ViewCommands.MinimizeWindow}" BorderBrush="{DynamicResource {x:Static ui:EnvironmentColors.MainWindowButtonActiveBorderBrushKey}}" BorderThickness="1,0,1,1" GlyphForeground="{DynamicResource {x:Static ui:EnvironmentColors.MainWindowButtonActiveGlyphBrushKey}}" HoverBackground="{DynamicResource {x:Static ui:EnvironmentColors.MainWindowButtonHoverActiveBrushKey}}" HoverBorderBrush="{DynamicResource {x:Static ui:EnvironmentColors.MainWindowButtonHoverActiveBorderBrushKey}}" HoverForeground="{DynamicResource {x:Static ui:EnvironmentColors.MainWindowButtonHoverActiveGlyphBrushKey}}" HoverBorderThickness="1,0,1,1" PressedBackground="{DynamicResource {x:Static ui:EnvironmentColors.MainWindowButtonDownBrushKey}}" PressedBorderBrush="{DynamicResource {x:Static ui:EnvironmentColors.MainWindowButtonDownBorderBrushKey}}" PressedForeground="{DynamicResource {x:Static ui:EnvironmentColors.MainWindowButtonDownGlyphBrushKey}}" PressedBorderThickness="1,0,1,1" Padding="0,3,0,0" Width="34" Height="26" AutomationProperties.Name="Minimize" AutomationProperties.AutomationId="Minimize" ToolTip="{x:Static vs:MainWindowResources.WindowMinimizeToolTip}" CommandParameter="{Binding RelativeSource={RelativeSource TemplatedParent}}">
        <Path Name="MinimizeButtonPath" x:Uid="MinimizeButtonPath" Width="9" Height="9" Stretch="None" Data="F1M0,6L0,9 9,9 9,6 0,6z" Fill="{Binding Path=(TextElement.Foreground), RelativeSource={RelativeSource Self}}" />
      </wcp:WindowTitleBarButton>
      <wcp:WindowTitleBarButton Name="MaximizeRestoreButton" x:Uid="MaximizeRestoreButton" VerticalAlignment="Top" Command="{x:Static vsc:ViewCommands.ToggleMaximizeRestoreWindow}" BorderBrush="{DynamicResource {x:Static ui:EnvironmentColors.MainWindowButtonActiveBorderBrushKey}}" BorderThickness="1,0,1,1" GlyphForeground="{DynamicResource {x:Static ui:EnvironmentColors.MainWindowButtonActiveGlyphBrushKey}}" HoverBackground="{DynamicResource {x:Static ui:EnvironmentColors.MainWindowButtonHoverActiveBrushKey}}" HoverBorderBrush="{DynamicResource {x:Static ui:EnvironmentColors.MainWindowButtonHoverActiveBorderBrushKey}}" HoverForeground="{DynamicResource {x:Static ui:EnvironmentColors.MainWindowButtonHoverActiveGlyphBrushKey}}" HoverBorderThickness="1,0,1,1" PressedBackground="{DynamicResource {x:Static ui:EnvironmentColors.MainWindowButtonDownBrushKey}}" PressedBorderBrush="{DynamicResource {x:Static ui:EnvironmentColors.MainWindowButtonDownBorderBrushKey}}" PressedForeground="{DynamicResource {x:Static ui:EnvironmentColors.MainWindowButtonDownGlyphBrushKey}}" PressedBorderThickness="1,0,1,1" Padding="0,3,0,0" Width="34" Height="26" AutomationProperties.Name="Maximize" AutomationProperties.AutomationId="Maximize" ToolTip="{x:Static vs:MainWindowResources.WindowMaximizeToolTip}" CommandParameter="{Binding RelativeSource={RelativeSource TemplatedParent}}">
        <Path Name="MaximizeRestoreButtonPath" x:Uid="MaximizeRestoreButtonPath" Width="9" Height="9" Stretch="Uniform" Data="F1M0,0L0,9 9,9 9,0 0,0 0,3 8,3 8,8 1,8 1,3z" Fill="{Binding Path=(TextElement.Foreground), RelativeSource={RelativeSource Self}}" />
      </wcp:WindowTitleBarButton>
      <wcp:WindowTitleBarButton Name="HideButton" x:Uid="HideButton" VerticalAlignment="Top" Command="{x:Static vsc:ViewCommands.CloseWindow}" BorderBrush="{DynamicResource {x:Static ui:EnvironmentColors.MainWindowButtonActiveBorderBrushKey}}" BorderThickness="1,0,1,1" GlyphForeground="{DynamicResource {x:Static ui:EnvironmentColors.MainWindowButtonActiveGlyphBrushKey}}" HoverBackground="{DynamicResource {x:Static ui:EnvironmentColors.MainWindowButtonHoverActiveBrushKey}}" HoverBorderBrush="{DynamicResource {x:Static ui:EnvironmentColors.MainWindowButtonHoverActiveBorderBrushKey}}" HoverForeground="{DynamicResource {x:Static ui:EnvironmentColors.MainWindowButtonHoverActiveGlyphBrushKey}}" HoverBorderThickness="1,0,1,1" PressedBackground="{DynamicResource {x:Static ui:EnvironmentColors.MainWindowButtonDownBrushKey}}" PressedBorderBrush="{DynamicResource {x:Static ui:EnvironmentColors.MainWindowButtonDownBorderBrushKey}}" PressedForeground="{DynamicResource {x:Static ui:EnvironmentColors.MainWindowButtonDownGlyphBrushKey}}" PressedBorderThickness="1,0,1,1" Padding="0,3,0,0" Width="34" Height="26" AutomationProperties.Name="Close" AutomationProperties.AutomationId="Close" ToolTip="{x:Static vs:MainWindowResources.WindowCloseToolTip}" CommandParameter="{Binding RelativeSource={RelativeSource Mode=FindAncestor, AncestorType={x:Type Window}}}">
        <Path Name="HideButtonPath" x:Uid="HideButtonPath" Width="10" Height="8" Stretch="Uniform" Data="F1M0,0L2,0 5,3 8,0 10,0 6,4 10,8 8,8 5,5 2,8 0,8 4,4 0,0z" Fill="{Binding Path=(TextElement.Foreground), RelativeSource={RelativeSource Self}}" />
      </wcp:WindowTitleBarButton>
    </StackPanel>
    <mwtb:FrameControlContainer Name="PART_TitleBarFrameControlContainer" x:Uid="PART_TitleBarFrameControlContainer" DockPanel.Dock="Right" TextElement.FontSize="{DynamicResource VsFont.EnvironmentFontSize}" TextElement.FontFamily="{DynamicResource VsFont.EnvironmentFontFamily}" Margin="0,0,2,0" DataContext="{Binding FrameControls}" />
    <TextBlock x:Uid="TextBlock_1" Text="{TemplateBinding Window.Title}" TextBlock.FontFamily="{DynamicResource VsFont.CaptionFontFamily}" TextBlock.FontSize="{DynamicResource VsFont.CaptionFontSize}" TextBlock.FontWeight="{DynamicResource VsFont.CaptionFontWeight}" TextTrimming="CharacterEllipsis" VerticalAlignment="Center" Margin="0,7,0,4" />
  </DockPanel>
</mwtb:MainWindowTitleBar>

Answer 1

My original answer was able to detect non-client mouse movements, but not when the mouse left. Based on the OP's finding of needing to use TrackMouseEvent to do this, I have updated my answer to show a fully-functioning example.

As mentioned in the comments, WPF does not handle/wrap non-client area events. I can find no explanation as to why. It is possible though to detect mouse movement (and therefore mouse enter) using a message hook.

From within a VS extension, the hook is initiated with this:

IntPtr vsHandle = System.Diagnostics.Process.GetCurrentProcess().MainWindowHandle;
HwndSource source = HwndSource.FromHwnd(vsHandle);
source.AddHook(new HwndSourceHook(WndProc));

The supporting code is:

private const int WM_NCHITTEST = 0x0084;
private const int WM_NCMOUSEMOVE = 0x00a0;
private const int WM_NCMOUSELEAVE = 0x02a2;

private enum HtResult
{
    HTERROR = (-2),
    HTTRANSPARENT = (-1),
    HTNOWHERE = 0,
    HTCLIENT = 1,
    HTCAPTION = 2,
    HTSYSMENU = 3,
    HTGROWBOX = 4,
    HTSIZE = HTGROWBOX,
    HTMENU = 5,
    HTHSCROLL = 6,
    HTVSCROLL = 7,
    HTMINBUTTON = 8,
    HTMAXBUTTON = 9,
    HTLEFT = 10,
    HTRIGHT = 11,
    HTTOP = 12,
    HTTOPLEFT = 13,
    HTTOPRIGHT = 14,
    HTBOTTOM = 15,
    HTBOTTOMLEFT = 16,
    HTBOTTOMRIGHT = 17,
    HTBORDER = 18,
    HTREDUCE = HTMINBUTTON,
    HTZOOM = HTMAXBUTTON,
    HTSIZEFIRST = HTLEFT,
    HTSIZELAST = HTBOTTOMRIGHT,
    HTOBJECT = 19,
    HTCLOSE = 20,
    HTHELP = 21
}

[Flags]
public enum TMEFlags : uint
{
    TME_CANCEL = 0x80000000,
    TME_HOVER = 0x00000001,
    TME_LEAVE = 0x00000002,
    TME_NONCLIENT = 0x00000010,
    TME_QUERY = 0x40000000
}

[DllImport("user32.dll")]
static extern int TrackMouseEvent(ref TRACKMOUSEEVENT lpEventTrack);

[StructLayout(LayoutKind.Sequential)]
public struct TRACKMOUSEEVENT
{
    public int cbSize;
    public TMEFlags dwFlags;
    public IntPtr hwndTrack;
    public int dwHoverTime;
}

private bool _trackingMouseMove;
private TRACKMOUSEEVENT _tme;

private IntPtr WndProc(IntPtr hwnd, int msg, IntPtr wParam, IntPtr lParam, ref bool handled)
{
    if (msg == WM_NCMOUSEMOVE)
    {
        if (!_trackingMouseMove)
        {
            _tme = new TRACKMOUSEEVENT();
            _tme.hwndTrack = hwnd;
            _tme.cbSize = Marshal.SizeOf(typeof(TRACKMOUSEEVENT));
            _tme.dwFlags = TMEFlags.TME_NONCLIENT | TMEFlags.TME_LEAVE;
            int success = TrackMouseEvent(ref _tme);
            _trackingMouseMove = (success != 0);
        }

        var hitTestResult = (HtResult)wParam;

        if ((hitTestResult == HtResult.HTSYSMENU) || (hitTestResult == HtResult.HTCAPTION))
        {
            // Raise event here
            System.Diagnostics.Debug.WriteLine("Mouse over title bar");
        }
    }
    else if (msg == WM_NCMOUSELEAVE)
    {
        _trackingMouseMove = false;
        System.Diagnostics.Debug.WriteLine("Mouse left the title bar");
    }

    return IntPtr.Zero;
}