2nd edit:
I think my original test script has an issue, the 10000000
times loop is, in fact, dealing with the same memory location of an array, which makes the unsafe
version (provided by Marc here) much faster than bitwise
version. I wrote another test script, I noticed that bitwise
and unsafe
offers almost the same performance.
loop for 10,000,000 times
Bitwise: 4218484; UnsafeRaw: 4101719
0.0284673328426447545529081831 (~2% Difference)
here is the code:
unsafe class UnsafeRaw
{
public static short ToInt16BigEndian(byte* buf)
{
return (short)ToUInt16BigEndian(buf);
}
public static int ToInt32BigEndian(byte* buf)
{
return (int)ToUInt32BigEndian(buf);
}
public static long ToInt64BigEndian(byte* buf)
{
return (long)ToUInt64BigEndian(buf);
}
public static ushort ToUInt16BigEndian(byte* buf)
{
return (ushort)((*buf++ << 8) | *buf);
}
public static uint ToUInt32BigEndian(byte* buf)
{
return unchecked((uint)((*buf++ << 24) | (*buf++ << 16) | (*buf++ << 8) | *buf));
}
public static ulong ToUInt64BigEndian(byte* buf)
{
unchecked
{
var x = (uint)((*buf++ << 24) | (*buf++ << 16) | (*buf++ << 8) | *buf++);
var y = (uint)((*buf++ << 24) | (*buf++ << 16) | (*buf++ << 8) | *buf);
return ((ulong)x << 32) | y;
}
}
}
class Bitwise
{
public static short ToInt16BigEndian(byte[] buffer, int beginIndex)
{
return unchecked((short)(buffer[beginIndex] << 8 | buffer[beginIndex + 1]));
}
public static int ToInt32BigEndian(byte[] buffer, int beginIndex)
{
return unchecked(buffer[beginIndex] << 24 | buffer[beginIndex + 1] << 16 | buffer[beginIndex + 2] << 8 | buffer[beginIndex + 3]);
}
public static long ToInt64BigEndian(byte[] buffer, int beginIndex)
{
return unchecked((long)buffer[beginIndex] << 56 | (long)buffer[beginIndex + 1] << 48 | (long)buffer[beginIndex + 2] << 40 | (long)buffer[beginIndex + 3] << 32 | (long)buffer[beginIndex + 4] << 24 | (long)buffer[beginIndex + 5] << 16 | (long)buffer[beginIndex + 6] << 8 | buffer[beginIndex + 7]);
}
public static ushort ToUInt16BigEndian(byte[] buffer, int beginIndex)
{
return unchecked((ushort)ToInt16BigEndian(buffer, beginIndex));
}
public static uint ToUInt32BigEndian(byte[] buffer, int beginIndex)
{
return unchecked((uint)ToInt32BigEndian(buffer, beginIndex));
}
public static ulong ToUInt64BigEndian(byte[] buffer, int beginIndex)
{
return unchecked((ulong)ToInt64BigEndian(buffer, beginIndex));
}
}
class BufferTest
{
static long LongRandom(long min, long max, Random rand)
{
long result = rand.Next((Int32)(min >> 32), (Int32)(max >> 32));
result = result << 32;
result = result | (long)rand.Next((Int32)min, (Int32)max);
return result;
}
public static void Main()
{
const int times = 10000000;
const int index = 100;
Random r = new Random();
Stopwatch sw1 = new Stopwatch();
Console.WriteLine($"loop for {times:##,###} times");
Thread.Sleep(1000);
for (int j = 0; j < times; j++)
{
short a = (short)r.Next(short.MinValue, short.MaxValue);
int b = r.Next(int.MinValue, int.MaxValue);
long c = LongRandom(int.MinValue, int.MaxValue, r);
ushort d = (ushort)r.Next(ushort.MinValue, ushort.MaxValue);
uint e = (uint)r.Next(int.MinValue, int.MaxValue);
ulong f = (ulong)LongRandom(int.MinValue, int.MaxValue, r);
var arr1 = BitConverter.GetBytes(a);
var arr2 = BitConverter.GetBytes(b);
var arr3 = BitConverter.GetBytes(c);
var arr4 = BitConverter.GetBytes(d);
var arr5 = BitConverter.GetBytes(e);
var arr6 = BitConverter.GetBytes(f);
Array.Reverse(arr1);
Array.Reverse(arr2);
Array.Reverse(arr3);
Array.Reverse(arr4);
Array.Reverse(arr5);
Array.Reverse(arr6);
var largerArr1 = new byte[1024];
var largerArr2 = new byte[1024];
var largerArr3 = new byte[1024];
var largerArr4 = new byte[1024];
var largerArr5 = new byte[1024];
var largerArr6 = new byte[1024];
Array.Copy(arr1, 0, largerArr1, index, arr1.Length);
Array.Copy(arr2, 0, largerArr2, index, arr2.Length);
Array.Copy(arr3, 0, largerArr3, index, arr3.Length);
Array.Copy(arr4, 0, largerArr4, index, arr4.Length);
Array.Copy(arr5, 0, largerArr5, index, arr5.Length);
Array.Copy(arr6, 0, largerArr6, index, arr6.Length);
sw1.Start();
var n1 = Bitwise.ToInt16BigEndian(largerArr1, index);
var n2 = Bitwise.ToInt32BigEndian(largerArr2, index);
var n3 = Bitwise.ToInt64BigEndian(largerArr3, index);
var n4 = Bitwise.ToUInt16BigEndian(largerArr4, index);
var n5 = Bitwise.ToUInt32BigEndian(largerArr5, index);
var n6 = Bitwise.ToUInt64BigEndian(largerArr6, index);
sw1.Stop();
//Console.WriteLine(n1 == a);
//Console.WriteLine(n2 == b);
//Console.WriteLine(n3 == c);
//Console.WriteLine(n4 == d);
//Console.WriteLine(n5 == e);
//Console.WriteLine(n6 == f);
}
Stopwatch sw2 = new Stopwatch();
for (int j = 0; j < times; j++)
{
short a = (short)r.Next(short.MinValue, short.MaxValue);
int b = r.Next(int.MinValue, int.MaxValue);
long c = LongRandom(int.MinValue, int.MaxValue, r);
ushort d = (ushort)r.Next(ushort.MinValue, ushort.MaxValue);
uint e = (uint)r.Next(int.MinValue, int.MaxValue);
ulong f = (ulong)LongRandom(int.MinValue, int.MaxValue, r);
var arr1 = BitConverter.GetBytes(a);
var arr2 = BitConverter.GetBytes(b);
var arr3 = BitConverter.GetBytes(c);
var arr4 = BitConverter.GetBytes(d);
var arr5 = BitConverter.GetBytes(e);
var arr6 = BitConverter.GetBytes(f);
Array.Reverse(arr1);
Array.Reverse(arr2);
Array.Reverse(arr3);
Array.Reverse(arr4);
Array.Reverse(arr5);
Array.Reverse(arr6);
var largerArr1 = new byte[1024];
var largerArr2 = new byte[1024];
var largerArr3 = new byte[1024];
var largerArr4 = new byte[1024];
var largerArr5 = new byte[1024];
var largerArr6 = new byte[1024];
Array.Copy(arr1, 0, largerArr1, index, arr1.Length);
Array.Copy(arr2, 0, largerArr2, index, arr2.Length);
Array.Copy(arr3, 0, largerArr3, index, arr3.Length);
Array.Copy(arr4, 0, largerArr4, index, arr4.Length);
Array.Copy(arr5, 0, largerArr5, index, arr5.Length);
Array.Copy(arr6, 0, largerArr6, index, arr6.Length);
sw2.Start();
unsafe
{
fixed (byte* p1 = &largerArr1[index], p2 = &largerArr2[index], p3 = &largerArr3[index], p4 = &largerArr4[index], p5 = &largerArr5[index], p6 = &largerArr6[index])
{
var u1 = UnsafeRaw.ToInt16BigEndian(p1);
var u2 = UnsafeRaw.ToInt32BigEndian(p2);
var u3 = UnsafeRaw.ToInt64BigEndian(p3);
var u4 = UnsafeRaw.ToUInt16BigEndian(p4);
var u5 = UnsafeRaw.ToUInt32BigEndian(p5);
var u6 = UnsafeRaw.ToUInt64BigEndian(p6);
//Console.WriteLine(u1 == a);
//Console.WriteLine(u2 == b);
//Console.WriteLine(u3 == c);
//Console.WriteLine(u4 == d);
//Console.WriteLine(u5 == e);
//Console.WriteLine(u6 == f);
}
}
sw2.Stop();
}
Console.WriteLine($"Bitwise: {sw1.ElapsedTicks}; UnsafeRaw: {sw2.ElapsedTicks}");
Console.WriteLine((decimal)sw1.ElapsedTicks / sw2.ElapsedTicks - 1);
Console.ReadKey();
}
}
1st edit:
I tried to implement using fixed
and stackalloc
, compare with bitwise
. It seems that bitwise is faster than unsafe
approach in my test code.
Here is the measurement:
10000000
times with stopwatch frequency 3515622
unsafe fixed
- 2239790 ticks
bitwise
- 672159 ticks
unsafe stackalloc
- 1624166 ticks
Is there anything I did wrong? I thought unsafe
will be faster than bitwise
.
Here is the code:
class Bitwise
{
public static short ToInt16BigEndian(byte[] buf, int i)
{
return (short)((buf[i] << 8) | buf[i + 1]);
}
public static int ToInt32BigEndian(byte[] buf, int i)
{
return (buf[i] << 24) | (buf[i + 1] << 16) | (buf[i + 2] << 8) | buf[i + 3];
}
public static long ToInt64BigEndian(byte[] buf, int i)
{
return (buf[i] << 56) | (buf[i + 1] << 48) | (buf[i + 2] << 40) | buf[i + 3] << 32 | (buf[i + 4] << 24) | (buf[i + 5] << 16) | (buf[i + 6] << 8) | buf[i + 7];
}
public static ushort ToUInt16BigEndian(byte[] buf, int i)
{
ushort value = 0;
for (var j = 0; j < 2; j++)
{
value = (ushort)unchecked((value << 8) | buf[j + i]);
}
return value;
}
public static uint ToUInt32BigEndian(byte[] buf, int i)
{
uint value = 0;
for (var j = 0; j < 4; j++)
{
value = unchecked((value << 8) | buf[j + i]);
}
return value;
}
public static ulong ToUInt64BigEndian(byte[] buf, int i)
{
ulong value = 0;
for (var j = 0; j < 8; j++)
{
value = unchecked((value << 8) | buf[i + j]);
}
return value;
}
}
class Unsafe
{
public static short ToInt16BigEndian(byte[] buf, int i)
{
byte[] arr = new byte[2];
arr[0] = buf[i + 1];
arr[1] = buf[i];
unsafe
{
fixed (byte* ptr = arr)
{
return *(short*)ptr;
}
}
}
public static int ToInt32BigEndian(byte[] buf, int i)
{
byte[] arr = new byte[4];
arr[0] = buf[i + 3];
arr[1] = buf[i + 2];
arr[2] = buf[i + 1];
arr[3] = buf[i];
unsafe
{
fixed (byte* ptr = arr)
{
return *(int*)ptr;
}
}
}
public static long ToInt64BigEndian(byte[] buf, int i)
{
byte[] arr = new byte[8];
arr[0] = buf[i + 7];
arr[1] = buf[i + 6];
arr[2] = buf[i + 5];
arr[3] = buf[i + 6];
arr[4] = buf[i + 3];
arr[5] = buf[i + 2];
arr[6] = buf[i + 1];
arr[7] = buf[i];
unsafe
{
fixed (byte* ptr = arr)
{
return *(long*)ptr;
}
}
}
public static ushort ToUInt16BigEndian(byte[] buf, int i)
{
byte[] arr = new byte[2];
arr[0] = buf[i + 1];
arr[1] = buf[i];
unsafe
{
fixed (byte* ptr = arr)
{
return *(ushort*)ptr;
}
}
}
public static uint ToUInt32BigEndian(byte[] buf, int i)
{
byte[] arr = new byte[4];
arr[0] = buf[i + 3];
arr[1] = buf[i + 2];
arr[2] = buf[i + 1];
arr[3] = buf[i];
unsafe
{
fixed (byte* ptr = arr)
{
return *(uint*)ptr;
}
}
}
public static ulong ToUInt64BigEndian(byte[] buf, int i)
{
byte[] arr = new byte[8];
arr[0] = buf[i + 7];
arr[1] = buf[i + 6];
arr[2] = buf[i + 5];
arr[3] = buf[i + 6];
arr[4] = buf[i + 3];
arr[5] = buf[i + 2];
arr[6] = buf[i + 1];
arr[7] = buf[i];
unsafe
{
fixed (byte* ptr = arr)
{
return *(ulong*)ptr;
}
}
}
}
class UnsafeAlloc
{
public static short ToInt16BigEndian(byte[] buf, int i)
{
unsafe
{
const int length = sizeof(short);
byte* arr = stackalloc byte[length];
byte* p = arr;
for (int j = length - 1; j >= 0; j--)
{
*p = buf[i + j];
p++;
}
return *(short*)arr;
}
}
public static int ToInt32BigEndian(byte[] buf, int i)
{
unsafe
{
const int length = sizeof(int);
byte* arr = stackalloc byte[length];
byte* p = arr;
for (int j = length - 1; j >= 0; j--)
{
*p = buf[i + j];
p++;
}
return *(int*)arr;
}
}
public static long ToInt64BigEndian(byte[] buf, int i)
{
unsafe
{
const int length = sizeof(long);
byte* arr = stackalloc byte[length];
byte* p = arr;
for (int j = length - 1; j >= 0; j--)
{
*p = buf[i + j];
p++;
}
return *(long*)arr;
}
}
public static ushort ToUInt16BigEndian(byte[] buf, int i)
{
unsafe
{
const int length = sizeof(ushort);
byte* arr = stackalloc byte[length];
byte* p = arr;
for (int j = length - 1; j >= 0; j--)
{
*p = buf[i + j];
p++;
}
return *(ushort*)arr;
}
}
public static uint ToUInt32BigEndian(byte[] buf, int i)
{
unsafe
{
const int length = sizeof(uint);
byte* arr = stackalloc byte[length];
byte* p = arr;
for (int j = length - 1; j >= 0; j--)
{
*p = buf[i + j];
p++;
}
return *(uint*)arr;
}
}
public static ulong ToUInt64BigEndian(byte[] buf, int i)
{
unsafe
{
const int length = sizeof(ulong);
byte* arr = stackalloc byte[length];
byte* p = arr;
for (int j = length - 1; j >= 0; j--)
{
*p = buf[i + j];
p++;
}
return *(ulong*)arr;
}
}
}
class Program
{
static void Main()
{
short a = short.MinValue + short.MaxValue / 2;
int b = int.MinValue + int.MaxValue / 2;
long c = long.MinValue + long.MaxValue / 2;
ushort d = ushort.MaxValue / 2;
uint e = uint.MaxValue / 2;
ulong f = ulong.MaxValue / 2;
Console.WriteLine(a);
Console.WriteLine(b);
Console.WriteLine(c);
Console.WriteLine(d);
Console.WriteLine(e);
Console.WriteLine(f);
Console.WriteLine();
var arr1 = BitConverter.GetBytes(a);
var arr2 = BitConverter.GetBytes(b);
var arr3 = BitConverter.GetBytes(c);
var arr4 = BitConverter.GetBytes(d);
var arr5 = BitConverter.GetBytes(e);
var arr6 = BitConverter.GetBytes(f);
Array.Reverse(arr1);
Array.Reverse(arr2);
Array.Reverse(arr3);
Array.Reverse(arr4);
Array.Reverse(arr5);
Array.Reverse(arr6);
Console.WriteLine(Unsafe.ToInt16BigEndian(arr1, 0));
Console.WriteLine(Unsafe.ToInt32BigEndian(arr2, 0));
Console.WriteLine(Unsafe.ToInt64BigEndian(arr3, 0));
Console.WriteLine(Unsafe.ToUInt16BigEndian(arr4, 0));
Console.WriteLine(Unsafe.ToUInt32BigEndian(arr5, 0));
Console.WriteLine(Unsafe.ToUInt64BigEndian(arr6, 0));
Console.WriteLine();
Console.WriteLine(Bitwise.ToInt16BigEndian(arr1, 0));
Console.WriteLine(Bitwise.ToInt32BigEndian(arr2, 0));
Console.WriteLine(Bitwise.ToInt64BigEndian(arr3, 0));
Console.WriteLine(Bitwise.ToUInt16BigEndian(arr4, 0));
Console.WriteLine(Bitwise.ToUInt32BigEndian(arr5, 0));
Console.WriteLine(Bitwise.ToUInt64BigEndian(arr6, 0));
Console.WriteLine();
Console.WriteLine(UnsafeAlloc.ToInt16BigEndian(arr1, 0));
Console.WriteLine(UnsafeAlloc.ToInt32BigEndian(arr2, 0));
Console.WriteLine(UnsafeAlloc.ToInt64BigEndian(arr3, 0));
Console.WriteLine(UnsafeAlloc.ToUInt16BigEndian(arr4, 0));
Console.WriteLine(UnsafeAlloc.ToUInt32BigEndian(arr5, 0));
Console.WriteLine(UnsafeAlloc.ToUInt64BigEndian(arr6, 0));
Console.WriteLine();
Stopwatch sw = new Stopwatch();
sw.Start();
int times = 10000000;
var t0 = sw.ElapsedTicks;
for (int i = 0; i < times; i++)
{
Unsafe.ToInt16BigEndian(arr1, 0);
Unsafe.ToInt32BigEndian(arr2, 0);
Unsafe.ToInt64BigEndian(arr3, 0);
Unsafe.ToUInt16BigEndian(arr4, 0);
Unsafe.ToUInt32BigEndian(arr5, 0);
Unsafe.ToUInt64BigEndian(arr6, 0);
}
var t1 = sw.ElapsedTicks;
var t2 = sw.ElapsedTicks;
for (int i = 0; i < times; i++)
{
Bitwise.ToInt16BigEndian(arr1, 0);
Bitwise.ToInt32BigEndian(arr2, 0);
Bitwise.ToInt64BigEndian(arr3, 0);
Bitwise.ToUInt16BigEndian(arr4, 0);
Bitwise.ToUInt32BigEndian(arr5, 0);
Bitwise.ToUInt64BigEndian(arr6, 0);
}
var t3 = sw.ElapsedTicks;
var t4 = sw.ElapsedTicks;
for (int i = 0; i < times; i++)
{
UnsafeAlloc.ToInt16BigEndian(arr1, 0);
UnsafeAlloc.ToInt32BigEndian(arr2, 0);
UnsafeAlloc.ToInt64BigEndian(arr3, 0);
UnsafeAlloc.ToUInt16BigEndian(arr4, 0);
UnsafeAlloc.ToUInt32BigEndian(arr5, 0);
UnsafeAlloc.ToUInt64BigEndian(arr6, 0);
}
var t5 = sw.ElapsedTicks;
Console.WriteLine($"{t1 - t0} {t3 - t2} {t5 - t4}");
Console.ReadKey();
}
public static string ByteArrayToString(byte[] ba)
{
return string.Concat(ba.Select(b => Convert.ToString(b, 2).PadLeft(8, '0')));
}
}
Original:
I'm confused about the bitwise way to convert c# data type short, int, long
and ushort, uint, ulong
to a byte array
, and vice versa.
Performance is really really important to me.
I know there is a slow way of doing all these using BitConverter
and Array.Reverse
, but the performance is terrible.
I know there are basically two more approaches other than BitConverter
, one is bitwise
, and the other is unsafe
.
After research on StackOverflow like:
Efficient way to read big-endian data in C#
Bitwise endian swap for various types
In C#, convert ulong[64] to byte[512] faster?
Fast string to byte[] conversion
I tried and tested bitwise
method first, combining all these small pieces into one whole picture.
15555
43425534
54354444354
432
234234
34324432234
15555
43425534
-1480130482 // wrong
432
234234
34324432234
I'm even more confused about all these bit shifting, here is my testing code:
class Program
{
static void Main()
{
short a = 15555;
int b = 43425534;
long c = 54354444354;
ushort d = 432;
uint e = 234234;
ulong f = 34324432234;
Console.WriteLine(a);
Console.WriteLine(b);
Console.WriteLine(c);
Console.WriteLine(d);
Console.WriteLine(e);
Console.WriteLine(f);
var arr1 = BitConverter.GetBytes(a);
var arr2 = BitConverter.GetBytes(b);
var arr3 = BitConverter.GetBytes(c);
var arr4 = BitConverter.GetBytes(d);
var arr5 = BitConverter.GetBytes(e);
var arr6 = BitConverter.GetBytes(f);
//Console.WriteLine(ByteArrayToString(arr1));
//Console.WriteLine(ByteArrayToString(arr2));
//Console.WriteLine(ByteArrayToString(arr3));
//Console.WriteLine(ByteArrayToString(arr4));
//Console.WriteLine(ByteArrayToString(arr5));
//Console.WriteLine(ByteArrayToString(arr6));
Array.Reverse(arr1);
Array.Reverse(arr2);
Array.Reverse(arr3);
Array.Reverse(arr4);
Array.Reverse(arr5);
Array.Reverse(arr6);
//Console.WriteLine(ByteArrayToString(arr1));
//Console.WriteLine(ByteArrayToString(arr2));
//Console.WriteLine(ByteArrayToString(arr3));
//Console.WriteLine(ByteArrayToString(arr4));
//Console.WriteLine(ByteArrayToString(arr5));
//Console.WriteLine(ByteArrayToString(arr6));
Console.WriteLine(ToInt16BigEndian(arr1, 0));
Console.WriteLine(ToInt32BigEndian(arr2, 0));
Console.WriteLine(ToInt64BigEndian(arr3, 0));
Console.WriteLine(ToUInt16BigEndian(arr4, 0));
Console.WriteLine(ToUInt32BigEndian(arr5, 0));
Console.WriteLine(ToUInt64BigEndian(arr6, 0));
Console.ReadKey();
}
public static string ByteArrayToString(byte[] ba)
{
return string.Concat(ba.Select(b => Convert.ToString(b, 2).PadLeft(8, '0')));
}
public static short ToInt16BigEndian(byte[] buf, int i)
{
return (short)((buf[i] << 8) | buf[i + 1]);
}
public static int ToInt32BigEndian(byte[] buf, int i)
{
return (buf[i] << 24) | (buf[i + 1] << 16) | (buf[i + 2] << 8) | buf[i + 3];
}
public static long ToInt64BigEndian(byte[] buf, int i)
{
return (buf[i] << 56) | (buf[i + 1] << 48) | (buf[i + 2] << 40) | (buf[i + 3] << 32) | (buf[i + 4] << 24) | (buf[i + 5] << 16) | (buf[i + 6] << 8) | buf[i + 7];
}
public static ushort ToUInt16BigEndian(byte[] buf, int i)
{
ushort value = 0;
for (var j = 0; j < 2; j++)
{
value = (ushort)unchecked((value << 8) | buf[j + i]);
}
return value;
}
public static uint ToUInt32BigEndian(byte[] buf, int i)
{
uint value = 0;
for (var j = 0; j < 4; j++)
{
value = unchecked((value << 8) | buf[j + i]);
}
return value;
}
public static ulong ToUInt64BigEndian(byte[] buf, int i)
{
ulong value = 0;
for (var j = 0; j < 8; j++)
{
value = unchecked((value << 8) | buf[i + j]);
}
return value;
}
}
I'm asking a solution that provides best performance, with consistent code style and cleanness of the code.
I'm asking a solution that provides best performance, with consistent code style and cleanness of the code.
Nope; pick any two. You can't have all three. For example, if you're after the best performance, then you may have to compromise on some of those other things.
In the future, Span<T>
(Span<byte>
) will be pretty useful for this scenario - with several IO APIs gaining support for Span<T>
and Memory<T>
- but for now your best bet is probably unsafe
code using a stackalloc byte*
(or using fixed
on a byte[]
) and writing to that directly, using either shifting or masking with offsets in the "other endian" case.
If you love us? You can donate to us via Paypal or buy me a coffee so we can maintain and grow! Thank you!
Donate Us With