delphi THashSHA2 return a wrong SHA256 on huge file

Question

Data.Cloud.CloudAPI.pas has class function TCloudSHA256Authentication.GetStreamToHashSHA256Hex(const Content: TStream): string; that return wrong SHA 256 on some file.

class function TCloudSHA256Authentication.GetStreamToHashSHA256Hex(const Content: TStream): string;
var
  LBytes : TBytes;
  Hash: THashSHA2;
begin
  LBytes := TBytesStream(Content).Bytes;
  //Hash bytes
  Hash := THashSHA2.Create;
  Hash.Update(LBytes);
  Result := Hash.HashAsString;
end;

AWS S3 return error:

The provided x-amz-content-sha256 header does not match what was computed

GetStreamToHashSHA256Hex seems produce a different sha256 from amazon:

<ClientComputedContentSHA256>f43ee89e2b7758057bb1f33eb8546d4c2c118f2ab932de89dbd74aabc0651053</ClientComputedContentSHA256>
<S3ComputedContentSHA256>3bbf5f864cc139cf6392b4623bd782a69d16929db713bffaa68035f8a5c3c0ce</S3ComputedContentSHA256>

I have made some tests wit a myfile.zip (600 MB) ...

TIdHashSHA256 an alternative from Indy return the right SHA256 (same of aws s3), eg.:

var
  aFileStream: TFileStream;
  aHash:       TIdHashSHA256;
begin
  aFileStream := TFileStream.Create('C:\myfile.zip', fmOpenRead or fmShareDenyWrite);
  aHash       := TIdHashSHA256.Create;

  try
    Result := aHash.HashStreamAsHex(aFileStream).ToLower;
  finally
     aFileStream.Free;
     aHash.Free;
  end;
end;

hash_file() from PHP return the right SHA256 (same of aws s3), eg.:

hash_file('sha256', 'C:\myfile.zip');

but THashSHA2 return a wrong sha256, eg.:

var
  LBytes : TBytes;
  Hash: THashSHA2;
begin
  LBytes := TFile.ReadAllBytes('C:\myfile.zip');
  Hash := THashSHA2.Create;
  Hash.Update(LBytes);
  Result := Hash.HashAsString;
end;

why?

UPDATE

this is my bug fix. Import Data.Cloud.CloudAPI.pas into the project and rewrite these function:

uses IdHash, IdHashSHA, IdSSLOpenSSL;

class function TCloudSHA256Authentication.GetHashSHA256Hex( HashString: string): string;
var
  aHash: TIdHashSHA256;
begin
  LoadOpenSSLLibrary;
  try
    if not(TIdHashSHA256.IsAvailable) then
      raise Exception.Create('HashSHA256 Isn''t available!');

    aHash := TIdHashSHA256.Create;
    try
      Result := aHash.HashStringAsHex(HashString).ToLower;
    finally
      aHash.Free;
    end;
  finally
    UnLoadOpenSSLLibrary;
  end;
end;

class function TCloudSHA256Authentication.GetStreamToHashSHA256Hex(const Content: TStream): string;
var
  aHash: TIdHashSHA256;
begin
  LoadOpenSSLLibrary;
  try
    if not(TIdHashSHA256.IsAvailable) then
      raise Exception.Create('HashSHA256 Isn''t available!');

    aHash := TIdHashSHA256.Create;
    try
      Result := aHash.HashStreamAsHex(Content).ToLower;
    finally
      aHash.Free;
    end;
  finally
    UnLoadOpenSSLLibrary;
  end;
end;

UPDATE 2

i have also try to implement the FredS suggestion, it works:

class function TCloudSHA256Authentication.GetHashSHA256Hex( HashString: string): string;
var
  Content: TStringStream;
  Hash: THashSHA2;
  LBytes: TArray<Byte>;
  Buffer: PByte;
  BufLen: Integer;
  Readed: Integer;
begin
  BufLen := 16 * 1024;

  Buffer  := AllocMem(BufLen);
  Hash    := THashSHA2.Create;
  Content := TStringStream.Create(HashString);
  try
    while Content.Position < Content.Size do
    begin
      Readed := Content.Read(Buffer^, BufLen);
      if Readed > 0 then
        Hash.update(Buffer^, Readed);
    end;
  finally
    Content.Free;
    FreeMem(Buffer);
  end;

  Result := Hash.HashAsString;
end;

class function TCloudSHA256Authentication.GetStreamToHashSHA256Hex(const Content: TStream): string;
var
  LBytes : TBytes;
  Hash: THashSHA2;
  Buffer: PByte;
  BufLen: Integer;
  Readed: Integer;
begin
  BufLen := 16 * 1024;

  Buffer := AllocMem(BufLen);
  Hash   := THashSHA2.Create;
  try
      Content.Seek(0, soFromBeginning);

      while Content.Position < Content.Size do
      begin
        Readed := Content.Read(Buffer^, BufLen);
        if Readed > 0 then
          Hash.update(Buffer^, Readed);
      end;

      Content.Seek(0, soFromBeginning);
  finally
      FreeMem(Buffer);
  end;

  Result := Hash.HashAsString;
end;

Answer 1

I just tested a +1.5 GB file using MS Cyrpto and THashSHA2 on Berlin, they both returned the same hash but MS Crypto like OpenSSL is much faster.

The problem is that the file is too large to hold in TBytes in one chunk. My record helper has TBytes.MaxLen = $F000; {61440} so you need to use a TFileStream and read the file in chunks into HashSHA2.Update instead.

Update: As per David Heffernan's comment I retested TBytes.MaxLen and it appears to be only limited by available memory.

---

Practical Example and Speed comparison between MS Crypto and Delphi HashSha2

Note: Requires Jedi API

  program SHA2SpeedTest;

  {$APPTYPE CONSOLE}
  {$R *.res}

  uses
    JwaWindows, Winapi.Windows, System.SysUtils, System.Classes, System.Diagnostics, System.Hash;

  const
    SHA256_LEN    = 256 div 8;
    ChunkSize     = $F000;

  type
    TBytesHelper = record helper for TBytes
    public
      function BinToHex: string;
    end;

  function TBytesHelper.BinToHex: string;
  var
    Len : Integer;
  begin
    Len := Length(Self);
    SetLength(Result, Len * 2));
    System.Classes.BinToHex(Self, PChar(Result), Len);
  end;

  procedure DelphiHash256(const AStream: TStream; out Bytes: TBytes);
  var
    HashSHA2: THashSHA2;
    BytesRead: Integer;
  begin
    HashSHA2 := THashSHA2.create;
    SetLength(Bytes, ChunkSize);
    AStream.Position := 0;
    repeat
      BytesRead := AStream.Read(Bytes, ChunkSize);
      if (BytesRead = 0) then Break; // Done
      HashSHA2.Update(Bytes, BytesRead);
    until False;
    Bytes := HashSHA2.HashAsBytes;
  end;


  function CryptoHash256(const AStream: TStream; out Bytes: TBytes): Boolean;
  var
    SigLen   : Cardinal;
    hHash    : HCRYPTHASH;
    hProv    : HCRYPTPROV;
    BytesRead: Integer;
  begin
    hProv  := 0; hHash  := 0;
    Result := False;

    If not CryptAcquireContext(hProv, nil, nil, PROV_RSA_AES, CRYPT_VERIFYCONTEXT) then Exit;
    try
      if not CryptCreateHash(hProv, CALG_SHA_256, 0, 0, hHash) then Exit;
      try
        SetLength(Bytes, ChunkSize);
        AStream.Position := 0;
        repeat
          BytesRead := AStream.Read(Bytes, ChunkSize);
          if (BytesRead = 0) then Break; // Done
          if not CryptHashData(hHash, @Bytes[0], BytesRead, 0) then Exit;
        until False;

        SigLen := SHA256_LEN;
        SetLength(Bytes, SigLen);
        Result := CryptGetHashParam(hHash, HP_HASHVAL, @Bytes[0], SigLen, 0);
      finally
        CryptDestroyHash(hHash);
      end;
    finally
      CryptReleaseContext(hProv, 0);
    end;
  end;

  var
    Stream: TStream;
    Bytes : TBytes;
    sw    : TStopwatch;
    CryptoTicks : int64;
    FileName : string;

    {* CheckFileName *}
    function CheckFileName: boolean;
    begin
      if (FileName='') then FileName := ParamStr(0);
      Result := FileExists(FileName);
      if not Result then Writeln('Invalid File name');
    end;
  begin
    repeat
      Writeln('Please Enter a valid File name, empty for this Executable');
      Readln(FileName);
    until CheckFileName;

    try
      Stream := TFileStream.Create(FileName, fmOpenRead + fmShareDenyNone);
      try
        WriteLn('Crypto - Calculating Checksum');
        sw.Start;
        if not CryptoHash256(Stream, Bytes) then raise Exception.Create('Something Happened :)');
        sw.Stop;
        Writeln(Bytes.BinToHex);
        WriteLn('Elapsed: ' + sw.Elapsed.ToString);
        CryptoTicks := sw.ElapsedTicks;

        WriteLn('Delphi - Calculating Checksum');
        sw.Reset; sw.Start;
        DelphiHash256(Stream, Bytes);
        sw.Stop;
        Writeln(Bytes.BinToHex);
        WriteLn('Elapsed: ' + sw.Elapsed.ToString);

        Writeln(Format('MS Crypto is %d%% faster', [(sw.ElapsedTicks-CryptoTicks) * 100 div CryptoTicks]));
      finally
        Stream.Free;
      end;
      Writeln('Hit <Enter> to exit');
      Readln;
    except
      on E: Exception do Writeln(E.ClassName, ': ', E.Message);
    end;

  end.