HttpClient header getting nulled when using in async methods

Question

I am using .NET Framework 4.6.1.

I have a controller in my web api, where I have static HttpClient to handle all the http requests. After I hosted my app on IIS, approximately once in a month, I get the following exception for all the incoming request to my app:

System.ArgumentNullException: Value cannot be null.
   at System.Threading.Monitor.Enter(Object obj)
   at System.Net.Http.Headers.HttpHeaders.ParseRawHeaderValues(String name, HeaderStoreItemInfo info, Boolean removeEmptyHeader)
   at System.Net.Http.Headers.HttpHeaders.AddHeaders(HttpHeaders sourceHeaders)
   at System.Net.Http.Headers.HttpRequestHeaders.AddHeaders(HttpHeaders sourceHeaders)
   at System.Net.Http.HttpClient.PrepareRequestMessage(HttpRequestMessage request)
   at System.Net.Http.HttpClient.SendAsync(HttpRequestMessage request, HttpCompletionOption completionOption, CancellationToken cancellationToken)
   at System.Net.Http.HttpClient.SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
   at System.Net.Http.HttpClient.PutAsync(Uri requestUri, HttpContent content, CancellationToken cancellationToken)
   at Attributes.Controllers.AttributesBaseController.<UpdateAttributes>d__6.MoveNext() in D:\Git\PortalSystem\Attributes\Controllers\AttributesBaseController.cs:line 42

If I restart the app pool on IIS, everything starts to work fine again. Here is the code that I have:

public class AttributesBaseController : ApiController
{
    [Inject]
    public IPortalsRepository PortalsRepository { get; set; }

    private static HttpClient Client = new HttpClient(new HttpClientHandler { Proxy = null, UseProxy = false })
                                                                            { Timeout = TimeSpan.FromSeconds(double.Parse(WebConfigurationManager.AppSettings["httpTimeout"])) };
    private static readonly Logger logger = LogManager.GetCurrentClassLogger();

    protected async Task UpdateAttributes(int clientId, int? updateAttrId = null)
    {
        try
        {
            Client.DefaultRequestHeaders.Accept.Clear();
            Client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));

            #region Update Client Dossier !!! BELOW IS LINE 42 !!!!          
            using (var response = await Client.PutAsync(new Uri(WebConfigurationManager.AppSettings["dossier"] + "api/dossier?clientId=" + clientId), null))
            {
                if (!response.IsSuccessStatusCode)
                {
                    logger.Error($"Dossier update failed");
                }
            }
            #endregion

            #region Gather Initial Info
            var checkSystems = PortalsRepository.GetCheckSystems(clientId);
            var currentAttributes = PortalsRepository.GetCurrentAttributes(clientId, checkSystems);
            #endregion

            List<Task> tasks = new List<Task>();
            #region Initialize Tasks
            foreach (var cs in checkSystems)
            {
                if (!string.IsNullOrEmpty(cs.KeyValue))
                {
                    tasks.Add(Task.Run(async () =>
                    {
                            var passedAttributes = currentAttributes.Where(ca => ca.SystemId == cs.SystemId && ca.AttributeId == cs.AttributeId && 
                            (ca.SysClientId == cs.KeyValue || ca.OwnerSysClientId == cs.KeyValue)).ToList();

                            if (cs.AttributeId == 2 && (updateAttrId == null || updateAttrId == 2))
                            {
                                await UpdateOpenWayIndividualCardsInfo(passedAttributes, cs, clientId);
                            }
                            else if (cs.AttributeId == 3 && (updateAttrId == null || updateAttrId == 3))
                            {
                                await UpdateEquationAccountsInfo(passedAttributes, cs, clientId);
                            }
                            else if (cs.AttributeId == 8 && (updateAttrId == null || updateAttrId == 8))
                            {
                                await UpdateOpenWayCorporateInfo(passedAttributes, cs, clientId);
                            }
                            else if (cs.AttributeId == 9 && (updateAttrId == null || updateAttrId == 9))
                            {
                                await UpdateEquationDealsInfo(passedAttributes, cs, clientId);
                            }
                            else if (cs.AttributeId == 10 && (updateAttrId == null || updateAttrId == 10))
                            {
                                await UpdateOpenWayIndividualCardDepositsInfo(passedAttributes, cs, clientId);
                            }
                            else if (cs.AttributeId == 16 && (updateAttrId == null || updateAttrId == 16))
                            {
                                await UpdateOpenWayBonusInfo(passedAttributes, cs, clientId);
                            }
                            else if (cs.AttributeId == 17 && (/*updateAttrId == null ||*/ updateAttrId == 17))
                            {
                                await UpdateExternalCardsInfo(passedAttributes, cs, clientId);
                            }
                            if (cs.AttributeId == 18 && (updateAttrId == null || updateAttrId == 18))
                            {
                                await UpdateCRSInfo(passedAttributes, cs, clientId);
                            }
                            else if (cs.AttributeId == 22 && (updateAttrId == null || updateAttrId == 22))
                            {
                                await UpdateCardInsuranceInfo(passedAttributes, cs, clientId);
                            }
                    }));
                }
            }
            #endregion

            // Run all tasks
            await Task.WhenAny(Task.WhenAll(tasks.ToArray()), Task.Delay(TimeSpan.FromSeconds(double.Parse(WebConfigurationManager.AppSettings["taskWaitTime"]))));
        }
        catch (Exception ex)
        {
            logger.Error(ex);
        }
    }
}

Can anyone give me advice/help to figure out the problem? I just don't know whether the problem is in the way I am using HttpClient with tasks or something bad happens on IIS.

Answer 1

Looking at the implementation of DefaultRequestHeaders, we can see that it uses a simple Dictionary to store the headers:

private Dictionary<string, HttpHeaders.HeaderStoreItemInfo> headerStore;

DefaultRequestHeaders.Accept.Clear just removes the key from the dictionary, without any kind of synchronization:

public bool Remove(string name)
{
  this.CheckHeaderName(name);
  if (this.headerStore == null)
    return false;
  return this.headerStore.Remove(name);
}

Dictionary.Remove isn't thread-safe, unpredictable behavior can happen if you access the dictionary during this operation.

Now if we look at the ParseRawHeaderValues method in your stacktrace:

private bool ParseRawHeaderValues(string name, HttpHeaders.HeaderStoreItemInfo info, bool removeEmptyHeader)
{
  lock (info)
  {
    // stuff
  }
  return true;
}

We can see that the error would be cause by info to be null. Now looking at the caller:

internal virtual void AddHeaders(HttpHeaders sourceHeaders)
{
  if (sourceHeaders.headerStore == null)
    return;
  List<string> stringList = (List<string>) null;
  foreach (KeyValuePair<string, HttpHeaders.HeaderStoreItemInfo> keyValuePair in sourceHeaders.headerStore)
  {
    if (this.headerStore == null || !this.headerStore.ContainsKey(keyValuePair.Key))
    {
      HttpHeaders.HeaderStoreItemInfo headerStoreItemInfo = keyValuePair.Value;
      if (!sourceHeaders.ParseRawHeaderValues(keyValuePair.Key, headerStoreItemInfo, false))
      {
        if (stringList == null)
          stringList = new List<string>();
        stringList.Add(keyValuePair.Key);
      }
      else
        this.AddHeaderInfo(keyValuePair.Key, headerStoreItemInfo);
    }
  }
  if (stringList == null)
    return;
  foreach (string key in stringList)
    sourceHeaders.headerStore.Remove(key);
}

Long story short, we iterate the dictionary in DefaultRequestHeaders (that's sourceHeaders.headerStore) and copy the headers into the request.

Summing it up, at the same time we have a thread iterating the contents of the dictionary, and another adding/removing elements. This can lead to the behavior you're seeing.

To fix this, you have two solutions:

1. Initialize DefaultRequestHeaders in a static constructor, then never change it:

   static AttributesBaseController 
   {
       Client = new HttpClient(new HttpClientHandler { Proxy = null, UseProxy = false })
       {
           Timeout = TimeSpan.FromSeconds(double.Parse(WebConfigurationManager.AppSettings["httpTimeout"]))
       };
   
       Client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
   }
   

2. Use SendAsync with your custom headers instead of PutAsync:

   var message = new HttpRequestMessage(HttpMethod.Put, new Uri(WebConfigurationManager.AppSettings["dossier"] + "api/dossier?clientId=" + clientId));
   message.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
   using (var response = await Client.SendAsync(message))
   {
        // ...
   }
   

---

Just for fun, a small repro:

var client = new HttpClient();

client.DefaultRequestHeaders.Accept.Clear();
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));

var storeField = typeof(HttpHeaders).GetField("headerStore", BindingFlags.Instance | BindingFlags.NonPublic);

FieldInfo valueField = null;

var store = (IEnumerable)storeField.GetValue(client.DefaultRequestHeaders);

foreach (var item in store)
{
    valueField = item.GetType().GetField("value", BindingFlags.Instance | BindingFlags.NonPublic);

    Console.WriteLine(valueField.GetValue(item));
}

for (int i = 0; i < 8; i++)
{
    Task.Run(() =>
    {
        int iteration = 0;

        while (true)
        {
            iteration++;

            try
            {
                foreach (var item in store)
                {
                    var value = valueField.GetValue(item);

                    if (value == null)
                    {
                        Console.WriteLine("Iteration {0}, value is null", iteration);
                    }

                    break;
                }

                client.DefaultRequestHeaders.Accept.Clear();
                client.DefaultRequestHeaders.Accept.Add(new Net.Http.Headers.MediaTypeWithQualityHeaderValue("application/json"));
            }
            catch (Exception) { }
        }
    });
}

Console.ReadLine();

Output:

System.Net.Http.Headers.HttpHeaders+HeaderStoreItemInfo

Iteration 137, value is null

Reproducing the issue may take a few tries because threads tend to get stuck in an infinite loop when accessing the dictionary concurrently (if it happens on your webserver, ASP.NET will abort the thread after the timeout elapses).