JSON object does not serialize for huge data

Question

I am trying to serialize data in JSON. But i face below exception.

OutOfMemoryException was unhandled by user code.
An exception of type 'System.OutOfMemoryException' occurred in Newtonsoft.Json.dll but was not handled in user code

Below i defined my code:

Main controller:

public class TrackingController : BaseAngularController
{
    var lstDetails = _services.GetTrackingDetailsByAWBIds(awbids, awbType);
    if (lstDetails != null)
    {
        return AngularJson(lstDetails);
    }
}

Base Controller:

public abstract class BaseAngularController : Controller
{
    public AngularJsonResult<T> AngularJson<T>(T model)
    {
        return new AngularJsonResult<T>() { Data = model };
    }
}

Angular JSON Result class:

public class AngularJsonResult<T> :AngularJsonResult
{
    public new T Data
    {
        get { return (T)base.Data; }
        set { base.Data = value; }
    }
}

JSON Result class:

public class AngularJsonResult : JsonResult
{
    public override void ExecuteResult(ControllerContext context)
    {
        DoUninterestingBaseClassStuff(context);

        SerializeData(context.HttpContext.Response);
    }

    private void DoUninterestingBaseClassStuff(ControllerContext context)
    {
        if (context == null)
        {
            throw new ArgumentNullException("context");
        }

        var response = context.HttpContext.Response;
        response.ContentType = string.IsNullOrEmpty(ContentType) ? "application/json" : ContentType;

        if (ContentEncoding != null)
        {
            response.ContentEncoding = ContentEncoding;
        }
    }

    protected virtual void SerializeData(HttpResponseBase response)
    {
        if (ErrorMessages.Any())
        {
            Data = new
            {
                ErrorMessage = string.Join("\n", ErrorMessages),
                ErrorMessages = ErrorMessages.ToArray()
            };

            response.StatusCode = 400;
        }

        if (Data == null) return;

        response.Write(Data.ToJson());
    }
}

Serializing Object to JSON:

public static class JsonExtensions
{
    public static string ToJson<T>(this T obj, bool includeNull = true)
    {
        var settings = new JsonSerializerSettings
        {
            ContractResolver = new CamelCasePropertyNamesContractResolver(),
            Converters = new JsonConverter[] { new StringEnumConverter() },
            ReferenceLoopHandling = Newtonsoft.Json.ReferenceLoopHandling.Ignore,//newly added
            //PreserveReferencesHandling =Newtonsoft.Json.PreserveReferencesHandling.Objects,
            NullValueHandling = includeNull ? NullValueHandling.Include : NullValueHandling.Ignore
        };
        return JsonConvert.SerializeObject(obj, settings);
    }
}

Here i defined AngularJson method for passing object and override ExecuteResult method for converting Object to JSON.

So my SerializeData method was passing Response and converting to Objet in JSON, like Data.ToJson()

Please let me know your suggestions.

Answer 1

Your problem is that you are serializing your huge data to a string in memory on the server, then writing the entire string to the HttpResponseBase (which also buffers everything by default), and running out of memory somewhere in the process, possibly by exceeding the maximum c# string length.

One way to reduce memory use is to serialize directly to HttpResponseBase.OutputStream using JsonSerializer.Serialize(). This avoids the intermediate string representation.

You may also need to set HttpResponseBase.Buffer = false, and if so, follow the advice given in Unbuffered Output Very Slow and wrap the output stream in a BufferedStream.

The following extension method can be used for this:

public static class HttpResponseBaseExtensions
{
    public static void WriteJson<T>(this HttpResponseBase response, T obj, bool useResponseBuffering = true, bool includeNull = true)
    {
        var contentEncoding = response.ContentEncoding ?? Encoding.UTF8;
        if (!useResponseBuffering)
        {
            response.Buffer = false;

            // use a BufferedStream as suggested in //https://stackoverflow.com/questions/26010915/unbuffered-output-very-slow
            var bufferedStream = new BufferedStream(response.OutputStream, 256 * 1024);
            bufferedStream.WriteJson(obj, contentEncoding, includeNull);
            bufferedStream.Flush();
        }
        else
        {
            response.OutputStream.WriteJson(obj, contentEncoding, includeNull);
        }
    }

    static void WriteJson<T>(this Stream stream, T obj, Encoding contentEncoding, bool includeNull)
    {
        var settings = new JsonSerializerSettings
        {
            ContractResolver = new CamelCasePropertyNamesContractResolver(),
            Converters = new JsonConverter[] { new StringEnumConverter() },
            ReferenceLoopHandling = Newtonsoft.Json.ReferenceLoopHandling.Ignore,//newly added
            //PreserveReferencesHandling =Newtonsoft.Json.PreserveReferencesHandling.Objects,
            NullValueHandling = includeNull ? NullValueHandling.Include : NullValueHandling.Ignore
        };
        var serializer = JsonSerializer.CreateDefault(settings);
        var textWriter = new StreamWriter(stream, contentEncoding);
        serializer.Serialize(textWriter, obj);
        textWriter.Flush();
    }
}

Then use the extension method in place of response.Write(Data.ToJson());