HttpWebRequest FileUpload causes OutOfMemoryException

Question

I always get this OutOfMemoryException:

 System.OutOfMemoryException was not handled.
 Message=Exception of type 'System.OutOfMemoryException' was thrown.
 Source=System
 StackTrace:
    at System.Net.ScatterGatherBuffers.AllocateMemoryChunk(Int32 newSize)
    at System.Net.ScatterGatherBuffers.Write(Byte[] buffer, Int32 offset, Int32 count)
    at System.Net.ConnectStream.InternalWrite(Boolean async, Byte[] buffer, Int32 offset, Int32 size, AsyncCallback callback, Object state)
    at System.Net.ConnectStream.Write(Byte[] buffer, Int32 offset, Int32 size)
    …

... and memory usage is steadily increasing (according to the Windows task manager) when I execute following source code:

Imports System.Net
Imports System.Text
Imports System.IO

Public Class HttpFileUploader

    Public Function uploadFile(ByVal containa As CookieContainer, ByVal uri As String, ByVal filePath As String, ByVal fileParameterName As String, ByVal contentType As String, ByVal otherParameters As Specialized.NameValueCollection) As String

        Dim boundary As String = "---------------------------" & DateTime.Now.Ticks.ToString("x")
        Dim newLine As String = System.Environment.NewLine
        Dim boundaryBytes As Byte() = System.Text.Encoding.ASCII.GetBytes(newLine & "--" & boundary & newLine)
        Dim request As Net.HttpWebRequest = Net.WebRequest.Create(uri)

        request.ContentType = "multipart/form-data; boundary=" & boundary
        request.Method = "POST"
        request.CookieContainer = containa
        request.AllowAutoRedirect = True
        request.Timeout = -1

        Using requestStream As IO.Stream = request.GetRequestStream()

            Dim formDataTemplate As String = "Content-Disposition: form-data; name=""{0}""{1}{1}{2}"

            For Each key As String In otherParameters.Keys

                requestStream.Write(boundaryBytes, 0, boundaryBytes.Length)
                Dim formItem As String = String.Format(formDataTemplate, key, newLine, otherParameters(key))
                Dim formItemBytes As Byte() = System.Text.Encoding.UTF8.GetBytes(formItem)
                requestStream.Write(formItemBytes, 0, formItemBytes.Length)

            Next key

            requestStream.Write(boundaryBytes, 0, boundaryBytes.Length)

            Dim headerTemplate As String = "Content-Disposition: form-data; name=""{0}""; filename=""{1}""{2}Content-Type: {3}{2}{2}"
            Dim header As String = String.Format(headerTemplate, fileParameterName, filePath, newLine, contentType)
            Dim headerBytes As Byte() = System.Text.Encoding.UTF8.GetBytes(header)
            requestStream.Write(headerBytes, 0, headerBytes.Length)

            Using fileStream As New IO.FileStream(filePath, IO.FileMode.Open, IO.FileAccess.Read)

                Dim buffer(4096) As Byte
                Dim bytesRead As Int32 = fileStream.Read(buffer, 0, buffer.Length)

                Do While (bytesRead > 0)
                    ' the exception is triggered by the following line:     
                    requestStream.Write(buffer, 0, bytesRead)
                    bytesRead = fileStream.Read(buffer, 0, buffer.Length)
                Loop

            End Using

            Dim trailer As Byte() = System.Text.Encoding.ASCII.GetBytes(newLine & "--" + boundary + "--" & newLine)
            requestStream.Write(trailer, 0, trailer.Length)
            requestStream.Close()

        End Using

        Dim response As Net.WebResponse = Nothing
        Dim responseText = ""

        Try

            response = request.GetResponse()

            Using responseStream As IO.Stream = response.GetResponseStream()

                Using responseReader As New IO.StreamReader(responseStream)

                    responseText = responseReader.ReadToEnd()

                End Using

            End Using

        Catch exception As Net.WebException

            MsgBox(exception.Message)

        Finally

            response.Close()
            response = Nothing
            request = Nothing

        End Try

        Return responseText

    End Function

End Class

I have marked with a comment the line that triggers the exception.

I'm using 10 threads for uploading different files (having sizes up to 250 MB).

Why am I running out of memory with this code?

Answer 1

Following code is working:

Imports System.Net
Imports System.Text
Imports System.IO

Public Class HttpFileUploader

    Public Function uploadFile(ByVal containa As CookieContainer, ByVal uri As String, ByVal filePath As String, ByVal fileParameterName As String, ByVal contentType As String, ByVal otherParameters As Specialized.NameValueCollection) As String

        Dim boundary As String = "---------------------------" & DateTime.Now.Ticks.ToString("x")
        Dim newLine As String = System.Environment.NewLine
        Dim boundaryBytes As Byte() = System.Text.Encoding.ASCII.GetBytes(newLine & "--" & boundary & newLine)
        Dim request As Net.HttpWebRequest = Net.WebRequest.Create(uri)

        request.ContentType = "multipart/form-data; boundary=" & boundary
        request.Method = "POST"
        request.CookieContainer = containa
        request.AllowAutoRedirect = True
        request.Timeout = -1
        request.KeepAlive = True
        request.AllowWriteStreamBuffering = False

        Dim ms As New MemoryStream()
        Dim formDataTemplate As String = "Content-Disposition: form-data; name=""{0}""{1}{1}{2}"

        For Each key As String In otherParameters.Keys
            ms.Write(boundaryBytes, 0, boundaryBytes.Length)
            Dim formItem As String = String.Format(formDataTemplate, key, newLine, otherParameters(key))
            Dim formItemBytes As Byte() = System.Text.Encoding.UTF8.GetBytes(formItem)
            ms.Write(formItemBytes, 0, formItemBytes.Length)
        Next key

        ms.Write(boundaryBytes, 0, boundaryBytes.Length)

        Dim headerTemplate As String = "Content-Disposition: form-data; name=""{0}""; filename=""{1}""{2}Content-Type: {3}{2}{2}"
        Dim header As String = String.Format(headerTemplate, fileParameterName, filePath, newLine, contentType)
        Dim headerBytes As Byte() = System.Text.Encoding.UTF8.GetBytes(header)
        ms.Write(headerBytes, 0, headerBytes.Length)

        Dim length As Long = ms.Length
        length += New FileInfo(filePath).Length
        request.ContentLength = length

        Using requestStream As IO.Stream = request.GetRequestStream()
            Dim bheader() As Byte = ms.ToArray()
            requestStream.Write(bheader, 0, bheader.Length)
            Using fileStream As New IO.FileStream(filePath, IO.FileMode.Open, IO.FileAccess.Read)

                Dim buffer(4096) As Byte
                Dim bytesRead As Int32 = fileStream.Read(buffer, 0, buffer.Length)

                Do While (bytesRead > 0)
                    requestStream.Write(buffer, 0, bytesRead)
                    bytesRead = fileStream.Read(buffer, 0, buffer.Length)
                Loop
            End Using
            requestStream.Close()
        End Using

        Dim response As Net.WebResponse = Nothing
        Dim responseText = ""

        Try

            response = request.GetResponse()

            Using responseStream As IO.Stream = response.GetResponseStream()

                Using responseReader As New IO.StreamReader(responseStream)

                    responseText = responseReader.ReadToEnd()

                End Using

            End Using

        Catch exception As Net.WebException

            MsgBox(exception.Message)

        Finally

            response.Close()
            response = Nothing
            request = Nothing
        End Try

        Return responseText

    End Function

End Class

Many many thanks to Tom and the guy from http://blogs.msdn.com/b/johan/archive/2006/11/15/are-you-getting-outofmemoryexceptions-when-uploading-large-files.aspx!!!