C# Quickest Way to Get Average Colors of Screen

Question

I'm currently working on creating an Ambilight for my computer monitor with C#, an arduino, and an Ikea Dioder. Currently the hardware portion runs flawlessly; however, I'm having a problem with detecting the average color of a section of screen.

I have two issues with the implementations that I'm using:

1. Performance - Both of these algorithms add a somewhat noticeable stutter to the screen. Nothing showstopping, but it's annoying while watching video.

2. No Fullscreen Game Support - When a game is in fullscreen mode both of these methods just return white.

   public class DirectxColorProvider : IColorProvider
   {
   
       private static Device d;
       private static Collection<long> colorPoints;
   
       public DirectxColorProvider()
       {
           PresentParameters present_params = new PresentParameters();
           if (d == null)
           {
               d = new Device(new Direct3D(), 0, DeviceType.Hardware, IntPtr.Zero, CreateFlags.SoftwareVertexProcessing, present_params);
           }
           if (colorPoints == null)
           {
               colorPoints = GetColorPoints();
           }
       }
   
       public byte[] GetColors()
       {
           var color = new byte[4];
   
           using (var screen = this.CaptureScreen())
           {
               DataRectangle dr = screen.LockRectangle(LockFlags.None);
               using (var gs = dr.Data)
               {
                   color = avcs(gs, colorPoints);
               }
           }
   
           return color;
       }
   
       private Surface CaptureScreen()
       {
           Surface s = Surface.CreateOffscreenPlain(d, Screen.PrimaryScreen.Bounds.Width, Screen.PrimaryScreen.Bounds.Height, Format.A8R8G8B8, Pool.Scratch);
           d.GetFrontBufferData(0, s);
           return s;
       }
   
       private static byte[] avcs(DataStream gs, Collection<long> positions)
       {
           byte[] bu = new byte[4];
           int r = 0;
           int g = 0;
           int b = 0;
           int i = 0;
   
           foreach (long pos in positions)
           {
               gs.Position = pos;
               gs.Read(bu, 0, 4);
               r += bu[2];
               g += bu[1];
               b += bu[0];
               i++;
           }
   
           byte[] result = new byte[3];
           result[0] = (byte)(r / i);
           result[1] = (byte)(g / i);
           result[2] = (byte)(b / i);
   
           return result;
       }
   
       private Collection<long> GetColorPoints()
       {
           const long offset = 20;
           const long Bpp = 4;
   
           var box = GetBox();
   
           var colorPoints = new Collection<long>();
           for (var x = box.X; x < (box.X + box.Length); x += offset)
           {
               for (var y = box.Y; y < (box.Y + box.Height); y += offset)
               {
                   long pos = (y * Screen.PrimaryScreen.Bounds.Width + x) * Bpp;
                   colorPoints.Add(pos);
               }
           }
   
           return colorPoints;
       }
   
       private ScreenBox GetBox()
       {
           var box = new ScreenBox();
   
           int m = 8;
   
           box.X = (Screen.PrimaryScreen.Bounds.Width - m) / 3;
           box.Y = (Screen.PrimaryScreen.Bounds.Height - m) / 3;
   
           box.Length = box.X * 2;
           box.Height = box.Y * 2;
   
           return box;
       }
   
       private class ScreenBox
       {
           public long X { get; set; }
           public long Y { get; set; }
           public long Length { get; set; }
           public long Height { get; set; }
       }
   
   }
   

You can find the file for the directX implmentation here.

public class GDIColorProvider : Form, IColorProvider
{
    private static Rectangle box;
    private readonly IColorHelper _colorHelper;

    public GDIColorProvider()
    {
        _colorHelper = new ColorHelper();
        box = _colorHelper.GetCenterBox();
    }

    public byte[] GetColors()
    {
        var colors = new byte[3];

        IntPtr hDesk = GetDesktopWindow();
        IntPtr hSrce = GetDC(IntPtr.Zero);
        IntPtr hDest = CreateCompatibleDC(hSrce);
        IntPtr hBmp = CreateCompatibleBitmap(hSrce, box.Width, box.Height);
        IntPtr hOldBmp = SelectObject(hDest, hBmp);
        bool b = BitBlt(hDest, box.X, box.Y, (box.Width - box.X), (box.Height - box.Y), hSrce, 0, 0, CopyPixelOperation.SourceCopy);
        using(var bmp = Bitmap.FromHbitmap(hBmp))
        {
            colors = _colorHelper.AverageColors(bmp);
        }

        SelectObject(hDest, hOldBmp);
        DeleteObject(hBmp);
        DeleteDC(hDest);
        ReleaseDC(hDesk, hSrce);

        return colors;
    }

    // P/Invoke declarations
    [DllImport("gdi32.dll")]
    static extern bool BitBlt(IntPtr hdcDest, int xDest, int yDest, int
    wDest, int hDest, IntPtr hdcSource, int xSrc, int ySrc, CopyPixelOperation rop);
    [DllImport("user32.dll")]
    static extern bool ReleaseDC(IntPtr hWnd, IntPtr hDc);
    [DllImport("gdi32.dll")]
    static extern IntPtr DeleteDC(IntPtr hDc);
    [DllImport("gdi32.dll")]
    static extern IntPtr DeleteObject(IntPtr hDc);
    [DllImport("gdi32.dll")]
    static extern IntPtr CreateCompatibleBitmap(IntPtr hdc, int nWidth, int nHeight);
    [DllImport("gdi32.dll")]
    static extern IntPtr CreateCompatibleDC(IntPtr hdc);
    [DllImport("gdi32.dll")]
    static extern IntPtr SelectObject(IntPtr hdc, IntPtr bmp);
    [DllImport("user32.dll")]
    private static extern IntPtr GetDesktopWindow();
    [DllImport("user32.dll")]
    private static extern IntPtr GetWindowDC(IntPtr ptr);
    [DllImport("user32.dll")]
    private static extern IntPtr GetDC(IntPtr ptr);
}

You Can Find the File for the GDI implementation Here.

The Full Codebase Can be Found Here.

Answer 1

Updated Answer

The problem of slow screen capture performance most likely is caused by BitBlt() doing a pixel conversion when the pixel formats of source and destination don't match. From the docs:

If the color formats of the source and destination device contexts do not match, the BitBlt function converts the source color format to match the destination format.

This is what caused slow performance in my code, especially in higher resolutions.

The default pixel format seems to be PixelFormat.Format32bppArgb, so that's what you should use for the buffer:

var screen = new Bitmap(bounds.Width, bounds.Height, PixelFormat.Format32bppArgb);
var gfx = Graphics.FromImage(screen);
gfx.CopyFromScreen(bounds.Location, new Point(0, 0), bounds.Size);

The next source for slow performance is Bitmap.GetPixel() which does boundary checks. Never use it when analyzing every pixel. Instead lock the bitmap data and get a pointer to it:

public unsafe Color GetAverageColor(Bitmap image, int sampleStep = 1) {
    var data = image.LockBits(
        new Rectangle(Point.Empty, Image.Size),
        ImageLockMode.ReadOnly,
        PixelFormat.Format32bppArgb);

    var row = (int*)data.Scan0.ToPointer();
    var (sumR, sumG, sumB) = (0L, 0L, 0L);
    var stride = data.Stride / sizeof(int) * sampleStep;

    for (var y = 0; y < data.Height; y += sampleStep) {
        for (var x = 0; x < data.Width; x += sampleStep) {
            var argb = row[x];
            sumR += (argb & 0x00FF0000) >> 16;
            sumG += (argb & 0x0000FF00) >> 8;
            sumB += argb & 0x000000FF;
        }
        row += stride;
    }

    image.UnlockBits(data);

    var numSamples = data.Width / sampleStep * data.Height / sampleStep;
    var avgR = sumR / numSamples;
    var avgG = sumG / numSamples;
    var avgB = sumB / numSamples;
    return Color.FromArgb((int)avgR, (int)avgG, (int)avgB);
}

This should get you well below 10 ms, depending on the screen size. In case it is still too slow you can increase the sampleStep parameter of GetAverageColor().

Original Answer

I recently did the same thing and came up with something that worked surprisingly good.

The trick is to create an additional bitmap that is 1x1 pixels in size and set a good interpolation mode on its graphics context (bilinear or bicubic, but not nearest neighbor).

Then you draw your captured bitmap into that 1x1 bitmap exploiting the interpolation and retrieve that pixel to get the average color.

I'm doing that at a rate of ~30 fps. When the screen shows a GPU rendering (e.g. watching YouTube full screen with enabled hardware acceleration in Chrome) there is no visible stuttering or anything. In fact, CPU utilization of the application is way below 10%. However, if I turn off Chrome's hardware acceleration then there is definitely some slight stuttering noticeable if you watch close enough.

Here are the relevant parts of the code:

using var screen = new Bitmap(width, height);
using var screenGfx = Graphics.FromImage(screen);

using var avg = new Bitmap(1, 1);
using var avgGfx = Graphics.FromImage(avg);
avgGfx.InterpolationMode = InterpolationMode.HighQualityBicubic;

while (true) {
    screenGfx.CopyFromScreen(left, top, 0, 0, screen.Size);
    avgGfx.DrawImage(screen, 0, 0, avg.Width, avg.Height);
    var color = avg.GetPixel(0, 0);
    var bright = (int)Math.Round(Math.Clamp(color.GetBrightness() * 100, 1, 100));
    // set color and brightness on your device
    // wait 1000/fps milliseconds
}

Note that this works for GPU renderings, because System.Drawing.Common uses GDI+ nowadays. However, it does not work when the content is DRM protected. So it won't work with Netflix for example :(

I published the code on GitHub. Even though I abandoned the project due to Netflix' DRM protection it might help someone else.