D3D9 Hook - Overlay with Direct3D9

Question

OK, so I'm trying to do some overlay for some extra buttons on a Direct X game.

I found a c++ sample that overlays quite nicely here: http://www.gamedev.net/topic/359794-c-direct3d-hooking-sample/

So I began to convert it to Delphi. With some logging I can see that it starts the hook on the correct process and hooks Direct3DCreate9() correctly.

Next TMyDirect3D9 is created successfully. But the process crashes from here.

My educated guess (based on some debugging in Ollydbg) that when I return MyDirect3D9 back to the original process via the hooked Direct3DCreate9() and it tries to call one of the class(interface) functions it fails.

Code follows. If I can give any other information to help let me know.

Main DLL:

library LeagueUtilityBox;

{$R *.res}

{$DEFINE DEBUG}

uses
  Windows,
  APIHijack in 'APIHijack.pas',
  Direct3D9 in '..\DirectX 9.0\Direct3D9.pas',
  uSharedMem in '..\Misc\uSharedMem.pas',
  MyDirect3D9 in 'MyDirect3D9.pas',
  MyDirect3DDevice9 in 'MyDirect3DDevice9.pas',
  {$IFDEF DEBUG}
  SysUtils,
  uLog in '..\Misc\uLog.pas',
  {$ENDIF}
  uMisc in 'uMisc.pas';

var
  SharedMem : TSharedMem;
  D3DHook: SDLLHook;
  hHook : DWORD;
  MyDirect3D9 : TMyDirect3D9;

function GetTargetProcess: String;
const
  KeyBase : DWORD = HKEY_CURRENT_USER;
  KeyLocation : String = 'Software\LeagueUtilityBox';
var
  RegKey : HKEY;
  TargetProcess : Array[0..511] Of Char;
  Count : DWORD;
begin
  Result := '';
  If RegOpenKeyEx(KeyBase, PChar(KeyLocation), 0, KEY_QUERY_VALUE, RegKey) = ERROR_SUCCESS Then
    begin
    Count := 512;
    If RegQueryValueEx(RegKey, nil, nil, nil, @TargetProcess[0], @Count) = ERROR_SUCCESS Then
      begin
      Result := String(TargetProcess);
    end;
  end;
end;

type
  TDirect3DCreate9 = function(SDKVersion: LongWord): Pointer; stdcall;

function MyDirect3DCreate9(SDKVersion: LongWord): Pointer; stdcall;
var
  OldFunc : TDirect3DCreate9;
  D3D : PIDirect3D9;
begin
  {$IFDEF DEBUG}
  WriteToLog('C:\LeagueUtilityBox.log', 'MyDirect3DCreate9 called');
  {$ENDIF}
  Result := nil;
  OldFunc := TDirect3DCreate9(D3DHook.Functions[0].OrigFn);
  D3D := OldFunc(SDKVersion);
  If D3D <> nil Then
    begin
    {$IFDEF DEBUG}
    WriteToLog('C:\LeagueUtilityBox.log', 'D3D created: 0x' + IntToHex(DWORD(Pointer(D3D)), 8));
    {$ENDIF}
    New(MyDirect3D9);
    MyDirect3D9 := TMyDirect3D9.Create(D3D);
    {$IFDEF DEBUG}
    WriteToLog('C:\LeagueUtilityBox.log', 'MyDirect3D9 Created');
    {$ENDIF}
    Result := @MyDirect3D9;
  end;
end;

procedure InitializeHook;
var
  Process : String;
  I : Integer;
begin
  SetLength(Process, 512);
  GetModuleFileName(GetModuleHandle(nil), PChar(Process), 512);
  For I := Length(Process) DownTo 1 Do
    begin
    If Process[I] = '\' Then Break;
  end;
  Process := Copy(Process, I + 1, Length(Process));
  If CompareString(LOCALE_SYSTEM_DEFAULT, NORM_IGNORECASE, PChar(GetTargetProcess), -1, PChar(Process), -1) = 2 Then
    begin
    {$IFDEF DEBUG}
    WriteToLog('C:\LeagueUtilityBox.log', 'Found target: ' + GetTargetProcess);
    {$ENDIF}
    With D3DHook Do
      begin
      Name := 'D3D9.DLL';
      UseDefault := False;
      DefaultFn := nil;
      SetLength(Functions, 1);
      Functions[0].Name := 'Direct3DCreate9';
      Functions[0].HookFn := @MyDirect3DCreate9;
      Functions[0].OrigFn := nil;
    end;
    {$IFDEF DEBUG}
    WriteToLog('C:\LeagueUtilityBox.log', 'About to hook: ' + String(AnsiString(D3DHook.Name)));
    {$ENDIF}
    HookAPICalls(@D3DHook);
    {$IFDEF DEBUG}
    WriteToLog('C:\LeagueUtilityBox.log', 'Hook completed: ' + String(AnsiString(D3DHook.Name)));
    {$ENDIF}
  end;
end;

procedure InitializeDLL;
begin
  SharedMem := TSharedMem.Create('LeagueUtilityBox', 1024);
  Try
    hHook := PDWORD(SharedMem.Buffer)^;
    {$IFDEF DEBUG}
    WriteToLog('C:\LeagueUtilityBox.log', 'Initializing DLL: ' + IntToStr(hHook));
    {$ENDIF}
  Finally
    SharedMem.Free;
  end;
end;

procedure UninitializeDLL;
begin
  UnhookWindowsHookEx(hHook);
end;

function WindowsHookCallback(nCode: Integer; WPARAM: Integer; LPARAM: Integer): LRESULT; stdcall;
begin
  Result := CallNextHookEx(hHook, nCode, WPARAM, LPARAM);
end;

procedure EntryPoint(Reason: DWORD);
begin
  Case Reason Of
    DLL_PROCESS_ATTACH:
      begin
      InitializeDLL;
      InitializeHook;
    end;
    DLL_PROCESS_DETACH:
      begin
      UninitializeDLL;
    end;
  end;
end;

exports
  WindowsHookCallback;

begin
  DLLProc := @EntryPoint;
  EntryPoint(DLL_PROCESS_ATTACH);
end.

The custom IDirect3D9:

unit MyDirect3D9;

interface

uses Direct3D9, Windows, uMisc, uLog;

type
  PMyDirect3D9 = ^TMyDirect3D9;
  TMyDirect3D9 = class(TInterfacedObject, IDirect3D9)
  private
    fD3D: PIDirect3D9;
  public
    constructor Create(D3D: PIDirect3D9);

    function QueryInterface(riid: REFIID; ppvObj: PPointer): HRESULT; stdcall;
    function _AddRef: DWORD; stdcall;
    function _Release: DWORD; stdcall;

    function RegisterSoftwareDevice(pInitializeFunction: Pointer): HResult; stdcall;
    function GetAdapterCount: LongWord; stdcall;
    function GetAdapterIdentifier(Adapter: LongWord; Flags: DWord; out pIdentifier: TD3DAdapterIdentifier9): HResult; stdcall;
    function GetAdapterModeCount(Adapter: LongWord; Format: TD3DFormat): LongWord; stdcall;
    function EnumAdapterModes(Adapter: LongWord; Format: TD3DFormat; Mode: LongWord; out pMode: TD3DDisplayMode): HResult; stdcall;
    function GetAdapterDisplayMode(Adapter: LongWord; out pMode: TD3DDisplayMode): HResult; stdcall;
    function CheckDeviceType(Adapter: LongWord; CheckType: TD3DDevType; AdapterFormat, BackBufferFormat: TD3DFormat; Windowed: BOOL): HResult; stdcall;
    function CheckDeviceFormat(Adapter: LongWord; DeviceType: TD3DDevType; AdapterFormat: TD3DFormat; Usage: DWord; RType: TD3DResourceType; CheckFormat: TD3DFormat): HResult; stdcall;
    function CheckDeviceMultiSampleType(Adapter: LongWord; DeviceType: TD3DDevType; SurfaceFormat: TD3DFormat; Windowed: BOOL; MultiSampleType: TD3DMultiSampleType; pQualityLevels: PDWORD): HResult; stdcall;
    function CheckDepthStencilMatch(Adapter: LongWord; DeviceType: TD3DDevType; AdapterFormat, RenderTargetFormat, DepthStencilFormat: TD3DFormat): HResult; stdcall;
    function CheckDeviceFormatConversion(Adapter: LongWord; DeviceType: TD3DDevType; SourceFormat, TargetFormat: TD3DFormat): HResult; stdcall;
    function GetDeviceCaps(Adapter: LongWord; DeviceType: TD3DDevType; out pCaps: TD3DCaps9): HResult; stdcall;
    function GetAdapterMonitor(Adapter: LongWord): HMONITOR; stdcall;
    function CreateDevice(Adapter: LongWord; DeviceType: TD3DDevType; hFocusWindow: HWND; BehaviorFlags: DWord; pPresentationParameters: PD3DPresentParameters; out ppReturnedDeviceInterface: IDirect3DDevice9): HResult; stdcall;
  end;

implementation

uses MyDirect3DDevice9;

constructor TMyDirect3D9.Create(D3D: PIDirect3D9);
begin
  {$IFDEF DEBUG}
  WriteToLog('C:\LeagueUtilityBox.log', 'TMyDirect3D9.Create');
  {$ENDIF}
  fD3D := D3D;
end;

function TMyDirect3D9.QueryInterface(riid: REFIID; ppvObj: PPointer): HRESULT; stdcall;
begin
  {$IFDEF DEBUG}
  WriteToLog('C:\LeagueUtilityBox.log', 'TMyDirect3D9.QueryInterface');
  {$ENDIF}
  Result := fD3D^.QueryInterface(riid, ppvObj);
end;

function TMyDirect3D9._AddRef: DWORD; stdcall;
begin
  {$IFDEF DEBUG}
  WriteToLog('C:\LeagueUtilityBox.log', 'TMyDirect3D9._AddRef');
  {$ENDIF}
  Result := fD3D^._AddRef;
end;

function TMyDirect3D9._Release: DWORD; stdcall;
var
  count : DWORD;
begin
  {$IFDEF DEBUG}
  WriteToLog('C:\LeagueUtilityBox.log', 'TMyDirect3D9._Release');
  {$ENDIF}
  count := fD3D^._Release;
  If count = 0 Then
    begin
    Self.Free;
  end;
  Result := count;
end;

function TMyDirect3D9.RegisterSoftwareDevice(pInitializeFunction: Pointer): HResult; stdcall;
begin
  {$IFDEF DEBUG}
  WriteToLog('C:\LeagueUtilityBox.log', 'TMyDirect3D9.RegisterSoftwareDevice');
  {$ENDIF}
  Result := fD3D^.RegisterSoftwareDevice(pInitializeFunction);
end;

function TMyDirect3D9.GetAdapterCount: LongWord; stdcall;
begin
  {$IFDEF DEBUG}
  WriteToLog('C:\LeagueUtilityBox.log', 'TMyDirect3D9.GetAdapterCount');
  {$ENDIF}
  Result := fD3D^.GetAdapterCount;
end;

function TMyDirect3D9.GetAdapterIdentifier(Adapter: LongWord; Flags: DWord; out pIdentifier: TD3DAdapterIdentifier9): HResult; stdcall;
begin
  {$IFDEF DEBUG}
  WriteToLog('C:\LeagueUtilityBox.log', 'TMyDirect3D9.GetAdapterIdentifier');
  {$ENDIF}
  Result := fD3D^.GetAdapterIdentifier(Adapter, Flags, pIdentifier);
end;

function TMyDirect3D9.GetAdapterModeCount(Adapter: LongWord; Format: TD3DFormat): LongWord; stdcall;
begin
  {$IFDEF DEBUG}
  WriteToLog('C:\LeagueUtilityBox.log', 'TMyDirect3D9.GetAdapterModeCount');
  {$ENDIF}
  Result := fD3D^.GetAdapterModeCount(Adapter, Format);
end;

function TMyDirect3D9.EnumAdapterModes(Adapter: LongWord; Format: TD3DFormat; Mode: LongWord; out pMode: TD3DDisplayMode): HResult; stdcall;
begin
  {$IFDEF DEBUG}
  WriteToLog('C:\LeagueUtilityBox.log', 'TMyDirect3D9.EnumAdapterModes');
  {$ENDIF}
  Result := fD3D^.EnumAdapterModes(Adapter, Format, Mode, pMode);
end;

function TMyDirect3D9.GetAdapterDisplayMode(Adapter: LongWord; out pMode: TD3DDisplayMode): HResult; stdcall;
begin
  {$IFDEF DEBUG}
  WriteToLog('C:\LeagueUtilityBox.log', 'TMyDirect3D9.GetAdapterDisplayMode');
  {$ENDIF}
  Result := fD3D^.GetAdapterDisplayMode(Adapter, pMode);
end;

function TMyDirect3D9.CheckDeviceType(Adapter: LongWord; CheckType: TD3DDevType; AdapterFormat, BackBufferFormat: TD3DFormat; Windowed: BOOL): HResult; stdcall;
begin
  {$IFDEF DEBUG}
  WriteToLog('C:\LeagueUtilityBox.log', 'TMyDirect3D9.CheckDeviceType');
  {$ENDIF}
  Result := fD3D^.CheckDeviceType(Adapter, CheckType, AdapterFormat, BackBufferFormat, Windowed);
end;

function TMyDirect3D9.CheckDeviceFormat(Adapter: LongWord; DeviceType: TD3DDevType; AdapterFormat: TD3DFormat; Usage: DWord; RType: TD3DResourceType; CheckFormat: TD3DFormat): HResult; stdcall;
begin
  {$IFDEF DEBUG}
  WriteToLog('C:\LeagueUtilityBox.log', 'TMyDirect3D9.CheckDeviceFormat');
  {$ENDIF}
  Result := fD3D^.CheckDeviceFormat(Adapter, DeviceType, AdapterFormat, Usage, RType, CheckFormat);
end;

function TMyDirect3D9.CheckDeviceMultiSampleType(Adapter: LongWord; DeviceType: TD3DDevType; SurfaceFormat: TD3DFormat; Windowed: BOOL; MultiSampleType: TD3DMultiSampleType; pQualityLevels: PDWORD): HResult; stdcall;
begin
  {$IFDEF DEBUG}
  WriteToLog('C:\LeagueUtilityBox.log', 'TMyDirect3D9.CheckDeviceMultiSampleType');
  {$ENDIF}
  Result := fD3D^.CheckDeviceMultiSampleType(Adapter, DeviceType, SurfaceFormat, Windowed, MultiSampleType, pQualityLevels);
end;

function TMyDirect3D9.CheckDepthStencilMatch(Adapter: LongWord; DeviceType: TD3DDevType; AdapterFormat, RenderTargetFormat, DepthStencilFormat: TD3DFormat): HResult; stdcall;
begin
  {$IFDEF DEBUG}
  WriteToLog('C:\LeagueUtilityBox.log', 'TMyDirect3D9.CheckDepthStencilMatch');
  {$ENDIF}
  Result := fD3D^.CheckDepthStencilMatch(Adapter, DeviceType, AdapterFormat, RenderTargetFormat, DepthStencilFormat);
end;

function TMyDirect3D9.CheckDeviceFormatConversion(Adapter: LongWord; DeviceType: TD3DDevType; SourceFormat, TargetFormat: TD3DFormat): HResult; stdcall;
begin
  {$IFDEF DEBUG}
  WriteToLog('C:\LeagueUtilityBox.log', 'TMyDirect3D9.CheckDeviceFormatConversion');
  {$ENDIF}
  Result := fD3D^.CheckDeviceFormatConversion(Adapter, DeviceType, SourceFormat, TargetFormat);
end;

function TMyDirect3D9.GetDeviceCaps(Adapter: LongWord; DeviceType: TD3DDevType; out pCaps: TD3DCaps9): HResult; stdcall;
begin
  {$IFDEF DEBUG}
  WriteToLog('C:\LeagueUtilityBox.log', 'TMyDirect3D9.GetDeviceCaps');
  {$ENDIF}
  Result := fD3D^.GetDeviceCaps(Adapter, DeviceType, pCaps);
end;

function TMyDirect3D9.GetAdapterMonitor(Adapter: LongWord): HMONITOR; stdcall;
begin
  {$IFDEF DEBUG}
  WriteToLog('C:\LeagueUtilityBox.log', 'TMyDirect3D9.GetAdapterMonitor');
  {$ENDIF}
  Result := fD3D^.GetAdapterMonitor(Adapter);
end;

function TMyDirect3D9.CreateDevice(Adapter: LongWord; DeviceType: TD3DDevType; hFocusWindow: HWND; BehaviorFlags: DWord; pPresentationParameters: PD3DPresentParameters; out ppReturnedDeviceInterface: IDirect3DDevice9): HResult; stdcall;
var
  hr : HRESULT;
begin
  {$IFDEF DEBUG}
  WriteToLog('C:\LeagueUtilityBox.log', 'TMyDirect3D9.CreateDevice');
  {$ENDIF}
  hr := fD3D^.CreateDevice(Adapter, DeviceType, hFocusWindow, BehaviorFlags, pPresentationParameters, ppReturnedDeviceInterface);
  If Succeeded(hr) Then
    begin
    {$IFDEF DEBUG}
    WriteToLog('C:\LeagueUtilityBox.log', 'fD3D^.CreateDevice Succeeded');
    {$ENDIF}
    ppReturnedDeviceInterface := TMyDirect3DDevice9.Create(PIDirect3D9(@Self), @ppReturnedDeviceInterface);
  end;
  Result := hr;
end;

end.

UPDATE: So, since the Delphi interfaces seem to act differently than a real one (Delphi has an in between for it to successfully talk to other interfaces). So I just converted the interface to an array of pointers.

Now the program successfully calls CreateDevice(). I can see this both in the logs and stepping through in Ollydbg.

Now what happens is that when CreateDevice calls the original IDirect3D9.CreateDevice() it crashes again. When I debug in Ollydbg I notice that it is dereferencing the pointer once too much.

UPDATE 2: Ok, fixed some pointer issues with PIDirect3D9 vs IDirect3D9 in different places. So the original IDirect3D9.CreateDevice() gets called. But it errors with D3DERR_INVALIDCALL!!

So confusing.

UPDATE 3: Ok, with some more debugging it seems that when I call the function an extra parameter gets pushed on the stack. Which makes the first param invalid. This is proved further by DirectX Debugging which says iAdapter parameter invalid (first param).

UPDATE 4: With using IntRefToMethPtr() to get the direct pointer to the original CreateDevice call I was able to get it to call with the stacks the same. Same result. It's looking like I've went the wrong route with trying to hook it in Delphi.

UPDATE 5: Rewrote the hooking method. Now I'm just hooking essentially EndScene(). Hook now works fine in a test program (Vertices.exe that came with the hook demo found in the first URL in this post). But in the main game it crashes the game. Either way I've learned a lot.

Answer 1

I've done this a few times now, and the details are a bit extensive to put in an answer, but there are a few common gotchas and a few specific ones that you'll need to work on.

First, you need to implement the IDirect3D9 and IDirect3DDevice9 interfaces (at least) exactly as they are done in the libraries, or binary-compatible. The interfaces are based on virtual function calls (not sure the Pascal equivalent), so all methods must be virtual, the methods should be in the same order and take the same arguments (which should also be in the same order), etc.

The part I'd look closely at is how pascal is handling the functions, they need to be binary-compatible with Visual C++-built code (__thiscall, virtual, etc).

Additionally, with a few simple regexes, you can usually generate your custom header and the skeleton of your code file from the existing D3D9 header. Note that while this might sound silly, a full-blown D3D9 wrapper can (for IDirect3DDevice9 alone) come out to 2300 lines; generating the basics from the header cuts out a lot of typing that might cause errors.

To handle buttons, you'll also need to a) draw on top of the existing render and b) catch input.

a) is trivial: you simply wait for device->Present() and do your drawing before calling the real present. The only real gotcha is device states. You'll need to save the existing states, set your states for overlay drawing, then reset the device states. Not resetting them properly causes all sorts of fun issues. The states that need set depend on your app, but typically culling, depth sort/test and such are the ones you want to disable.

b) to do buttons, you'll need to also hook into the window's input somehow. Implementing the same sort of wrapper you have here, but for DInput (if its used) is probably a good idea. You can then perform your input check, rendering and logic in the I...Device9 wrapper's Present method.

There is also a decent bit of code around for wrappers like these; I have full d3d 8-to-9 (runtime translation) and 9, and I know of 2 other d3d9 wrappers that can be reused. That might be worth looking into, to either check your code or use existing code.

If there is more info you're interested in as regards to this, or anything interesting you find, I'd be happy to help/like to know.

Answer 2

You might want to check out existing DirextX packages for Delphi, even just to confirm (with their examples) that the constructs used are the same as one that you use.

The site I know best is Clootie's: http://clootie.ru/delphi/index.html But afaik there are multiple attempts

There are both DX9 and DX10 SDKs with examples there.

Answer 3

You can get a fully working D3D9 Proxy DLL if you take the clootie's D3D9 SDK

http://sourceforge.net/projects/delphi-dx9sdk/

and the "advanced" d3d9 base from GD

http://www.gamedeception.net/attachment.php?attachmentid=3035&d=1260299029

I use it on Delphi Architect XE3 and compiles and workes fine.