Capture screenshot of fullscreen DX11 program using SharpDX and EasyHook

Question

Before anybody mentions it, I refered to this link to find out how I needed to copy the backbuffer to a bitmap.

Current situation

• I am injected to the target process
• Target process' FeatureLevel = Level_11_0
• Target SwapChain is being made with DXGI_SWAP_CHAIN_FLAG_ALLOW_MODE_SWITCH flag.
• SwapChain::Present function is hooked.
• Screenshot turns out black and target process crashes. without screenshot process runs fine.

Desired situation

Make the screenshot properly and let the target process continue with its normal execution.

Code

NOTE Hook class is the same as in the link. I only added an UnmodifiableHook version of it which does what its name says. I left out all unimportant bits.

TestSwapChainHook.cs

using System;
using System.Runtime.InteropServices;

namespace Test
{
    public sealed class TestSwapChainHook : IDisposable
    {
        private enum IDXGISwapChainVirtualTable
        {
            QueryInterface = 0,
            AddRef = 1,
            Release = 2,
            SetPrivateData = 3,
            SetPrivateDataInterface = 4,
            GetPrivateData = 5,
            GetParent = 6,
            GetDevice = 7,
            Present = 8,
            GetBuffer = 9,
            SetFullscreenState = 10,
            GetFullscreenState = 11,
            GetDesc = 12,
            ResizeBuffers = 13,
            ResizeTarget = 14,
            GetContainingOutput = 15,
            GetFrameStatistics = 16,
            GetLastPresentCount = 17,
        }

        public static readonly int VIRTUAL_METHOD_COUNT_LEVEL_DEFAULT = 18;

        private static IntPtr[] SWAP_CHAIN_VIRTUAL_TABLE_ADDRESSES;

        [UnmanagedFunctionPointer(CallingConvention.StdCall, CharSet = CharSet.Unicode, SetLastError = true)]
        public delegate int DXGISwapChainPresentDelegate(IntPtr thisPtr, uint syncInterval, SharpDX.DXGI.PresentFlags flags);

        public delegate int DXGISwapChainPresentHookDelegate(UnmodifiableHook<DXGISwapChainPresentDelegate> hook, IntPtr thisPtr, uint syncInterval, SharpDX.DXGI.PresentFlags flags);

        private DXGISwapChainPresentHookDelegate _present;
        private Hook<DXGISwapChainPresentDelegate> presentHook;

        static TestSwapChainHook()
        {
            SharpDX.DXGI.Rational rational = new SharpDX.DXGI.Rational(60, 1);
            SharpDX.DXGI.ModeDescription modeDescription = new SharpDX.DXGI.ModeDescription(100, 100, rational, SharpDX.DXGI.Format.R8G8B8A8_UNorm);
            SharpDX.DXGI.SampleDescription sampleDescription = new SharpDX.DXGI.SampleDescription(1, 0);

            using (SharpDX.Windows.RenderForm renderForm = new SharpDX.Windows.RenderForm())
            {
                SharpDX.DXGI.SwapChainDescription swapChainDescription = new SharpDX.DXGI.SwapChainDescription();
                swapChainDescription.BufferCount = 1;
                swapChainDescription.Flags = SharpDX.DXGI.SwapChainFlags.None;
                swapChainDescription.IsWindowed = true;
                swapChainDescription.ModeDescription = modeDescription;
                swapChainDescription.OutputHandle = renderForm.Handle;
                swapChainDescription.SampleDescription = sampleDescription;
                swapChainDescription.SwapEffect = SharpDX.DXGI.SwapEffect.Discard;
                swapChainDescription.Usage = SharpDX.DXGI.Usage.RenderTargetOutput;

                SharpDX.Direct3D11.Device device = null;
                SharpDX.DXGI.SwapChain swapChain = null;
                SharpDX.Direct3D11.Device.CreateWithSwapChain(SharpDX.Direct3D.DriverType.Hardware, SharpDX.Direct3D11.DeviceCreationFlags.BgraSupport, swapChainDescription, out device, out swapChain);
                try
                {
                    IntPtr swapChainVirtualTable = Marshal.ReadIntPtr(swapChain.NativePointer);

                    SWAP_CHAIN_VIRTUAL_TABLE_ADDRESSES = new IntPtr[VIRTUAL_METHOD_COUNT_LEVEL_DEFAULT];
                    for (int x = 0; x < VIRTUAL_METHOD_COUNT_LEVEL_DEFAULT; x++)
                    {
                        SWAP_CHAIN_VIRTUAL_TABLE_ADDRESSES[x] = Marshal.ReadIntPtr(swapChainVirtualTable, x * IntPtr.Size);
                    }

                    device.Dispose();
                    swapChain.Dispose();
                }
                catch (Exception)
                {
                    if (device != null)
                    {
                        device.Dispose();
                    }

                    if (swapChain != null)
                    {
                        swapChain.Dispose();
                    }

                    throw;
                }
            }
        }

        public TestSwapChainHook()
        {
            this._present = null;

            this.presentHook = new Hook<DXGISwapChainPresentDelegate>(
                        SWAP_CHAIN_VIRTUAL_TABLE_ADDRESSES[(int)IDXGISwapChainVirtualTable.Present],
                        new DXGISwapChainPresentDelegate(hookPresent),
                        this);
        }

        public void activate()
        {
            this.presentHook.activate();
        }

        public void deactivate()
        {
            this.presentHook.deactivate();
        }

        private int hookPresent(IntPtr thisPtr, uint syncInterval, SharpDX.DXGI.PresentFlags flags)
        {
            lock (this.presentHook)
            {
                if (this._present == null)
                {
                    return this.presentHook.original(thisPtr, syncInterval, flags);
                }
                else
                {
                    return this._present(new UnmodifiableHook<DXGISwapChainPresentDelegate>(this.presentHook), thisPtr, syncInterval, flags);
                }
            }
        }

        public DXGISwapChainPresentHookDelegate present
        {
            get
            {
                lock (this.presentHook)
                {
                    return this._present;
                }
            }
            set
            {
                lock (this.presentHook)
                {
                    this._present = value;
                }
            }
        }
    }
}

Using code

initialization

private TestSwapChain swapChainHook;
private bool capture = false;
private object captureLock = new object();

this.swapChainHook = new TestSwapChainHook();
this.swapChainHook.present = presentHook;
this.swapChainHook.activate();

EDIT

I used a different method to capture a screenshot described in this link. However my screenshot turns out like this:

Now this seems to be a problem with my conversion settings or whatever but I'm unable to find out what exactly I need to do to fix it. I know that the surface I'm converting to a bitmap uses the DXGI_FORMAT_R10G10B10A2_UNORM format (32-bits, 10 bits per color and 2 for alpha I think?). But I'm not sure how this even works in the for loops (skipping bytes and stuff). I just plain copy pasted it.

new hook function

private int presentHook(UnmodifiableHook<IDXGISwapChainHook.DXGISwapChainPresentDelegate> hook, IntPtr thisPtr, uint syncInterval, SharpDX.DXGI.PresentFlags flags)
{
    try
    {
        lock (this.captureLock)
        {
            if (this.capture)
            {
                SharpDX.DXGI.SwapChain swapChain = (SharpDX.DXGI.SwapChain)thisPtr;

                using (SharpDX.Direct3D11.Texture2D backBuffer = swapChain.GetBackBuffer<SharpDX.Direct3D11.Texture2D>(0))
                {
                    SharpDX.Direct3D11.Texture2DDescription texture2DDescription = backBuffer.Description;
                    texture2DDescription.CpuAccessFlags = SharpDX.Direct3D11.CpuAccessFlags.Read;
                    texture2DDescription.Usage = SharpDX.Direct3D11.ResourceUsage.Staging;
                    texture2DDescription.OptionFlags = SharpDX.Direct3D11.ResourceOptionFlags.None;
                    texture2DDescription.BindFlags = SharpDX.Direct3D11.BindFlags.None;

                    using (SharpDX.Direct3D11.Texture2D texture = new SharpDX.Direct3D11.Texture2D(backBuffer.Device, texture2DDescription))
                    {
                        //DXGI_FORMAT_R10G10B10A2_UNORM
                        backBuffer.Device.ImmediateContext.CopyResource(backBuffer, texture);

                        using (SharpDX.DXGI.Surface surface = texture.QueryInterface<SharpDX.DXGI.Surface>())
                        {
                            SharpDX.DataStream dataStream;
                            SharpDX.DataRectangle map = surface.Map(SharpDX.DXGI.MapFlags.Read, out dataStream);
                            try
                            {
                                byte[] pixelData = new byte[surface.Description.Width * surface.Description.Height * 4];
                                int lines = (int)(dataStream.Length / map.Pitch);
                                int dataCounter = 0;
                                int actualWidth = surface.Description.Width * 4;

                                for (int y = 0; y < lines; y++)
                                {
                                    for (int x = 0; x < map.Pitch; x++)
                                    {
                                        if (x < actualWidth)
                                        {
                                            pixelData[dataCounter++] = dataStream.Read<byte>();
                                        }
                                        else
                                        {
                                            dataStream.Read<byte>();
                                        }
                                    }
                                }

                                GCHandle handle = GCHandle.Alloc(pixelData, GCHandleType.Pinned);
                                try
                                {
                                    using (Bitmap bitmap = new Bitmap(surface.Description.Width, surface.Description.Height, map.Pitch, PixelFormat.Format32bppArgb, handle.AddrOfPinnedObject()))
                                    {
                                        bitmap.Save(@"C:\Users\SOMEUSERNAME\Desktop\test.bmp");
                                    }
                                }
                                finally
                                {
                                    if (handle.IsAllocated)
                                    {
                                        handle.Free();
                                    }
                                }
                            }
                            finally
                            {
                                surface.Unmap();

                                dataStream.Dispose();
                            }
                        }
                    }
                }

                this.capture = false;
            }
        }
    }
    catch(Exception ex)
    {
        MessageBox.Show(ex.ToString());
    }

    return hook.original(thisPtr, syncInterval, flags);
}

Answer

Turns out the DXGI_FORMAT_R10G10B10A2_UNORM format is in this bit format:

A=alpha
B=blue
G=green
R=red

AABBBBBB BBBBGGGG GGGGGGRR RRRRRRRR

And Format32bppArgb is in this byte order:

BGRA

So the final loop code would be:

while (pixelIndex < pixelData.Length)
{
    uint currentPixel = dataStream.Read<uint>();

    uint r = (currentPixel & 0x3FF);
    uint g = (currentPixel & 0xFFC00) >> 10;
    uint b = (currentPixel & 0x3FF00000) >> 20;
    uint a = (currentPixel & 0xC0000000) >> 30;

    pixelData[pixelIndex++] = (byte)(b >> 2);
    pixelData[pixelIndex++] = (byte)(g >> 2);
    pixelData[pixelIndex++] = (byte)(r >> 2);
    pixelData[pixelIndex++] = (byte)(a << 6);

    while ((pixelIndex % map.Pitch) >= actualWidth)
    {
        dataStream.Read<byte>();
        pixelIndex++;
    }
}

Answer 1

That screenshot does look like R10G10B10A2 is getting stuffed into R8G8B8A8. I haven't tested your code but we should have this bit layout

xxxxxxxx yyyyyyyy zzzzzzzz wwwwwwww
RRRRRRRR RRGGGGGG GGGGBBBB BBBBBBAA

and you can extract them as follows

byte x = data[ptr++];
byte y = data[ptr++];
byte z = data[ptr++];
byte w = data[ptr++];

int r = x << 2 | y >> 6;
int g = (y & 0x3F) << 4 | z >> 4;
int b = (z & 0xF) << 6 | w >> 2;
int a = w & 0x3;

where r, g, b now have 10 bit resolution. If you want to scale them back to bytes you can do that with (byte)(r >> 2).

Update

This would replace your double for loop. I have no way of testing this so I don't want to push it further, but I believe the idea is correct. The last check should skip the padding bytes in each row.

while(dataCounter < pixelData.Length)
{

    byte x = dataStream.Read<byte>();
    byte y = dataStream.Read<byte>();
    byte z = dataStream.Read<byte>();
    byte w = dataStream.Read<byte>();

    int r = x << 2 | y >> 6;
    int g = (y & 0x3F) << 4 | z >> 4;
    int b = (z & 0xF) << 6 | w >> 2;
    int a = w & 0x3;

    pixelData[dataCounter++] = (byte)(r >> 2);
    pixelData[dataCounter++] = (byte)(g >> 2);
    pixelData[dataCounter++] = (byte)(b >> 2);
    pixelData[dataCounter++] = (byte)(a << 6);

    while((dataCounter % map.Pitch) >= actualWidth)
    {
        dataStream.Read<byte>();
        dataCounter++;
    }

}