.NET Core/System.Text.Json: Enumerate and add/replace json properties/values

Question

In an earlier question of mine I asked how to populate an existing object using System.Text.Json.

One of the great answers showed a solution parsing the json string with JsonDocument and enumerate it with EnumerateObject.

Over time my json string evolved and does now also contain an array of objects, and when parsing that with the code from the linked answer it throws the following exception:

The requested operation requires an element of type 'Object', but the target element has type 'Array'.

I figured out that one can in one way or the other look for the JsonValueKind.Array, and do something like this

if (json.ValueKind.Equals(JsonValueKind.Array))
{
    foreach (var item in json.EnumerateArray())
    {
        foreach (var property in item.EnumerateObject())
        {
            await OverwriteProperty(???);
        }
    }
}

but I can't make that work.

How to do this, and as a generic solution?

I would like to get "Result 1", where array items gets added/updated, and "Result 2" (when passing a variable), where the whole array gets replaced.

For "Result 2" I assume one can detect if (JsonValueKind.Array)) in the OverwriteProperty method, and where/how to pass the "replaceArray" variable? ... while iterating the array or the objects?

Some sample data:

Json string initial

{
  "Title": "Startpage",
  "Links": [
    {
      "Id": 10,
      "Text": "Start",
      "Link": "/index"
    },
    {
      "Id": 11,
      "Text": "Info",
      "Link": "/info"
    }
  ]
}

Json string to add/update

{
  "Head": "Latest news",
  "Links": [
    {
      "Id": 11,
      "Text": "News",
      "Link": "/news"
    },
    {
      "Id": 21,
      "Text": "More News",
      "Link": "/morenews"
    }
  ]
}

Result 1

{
  "Title": "Startpage",
  "Head": "Latest news"
  "Links": [
    {
      "Id": 10,
      "Text": "Start",
      "Link": "/indexnews"
    },
    {
      "Id": 11,
      "Text": "News",
      "Link": "/news"
    },
    {
      "Id": 21,
      "Text": "More news",
      "Link": "/morenews"
    }
  ]
}

Result 2

{
  "Title": "Startpage",
  "Head": "Latest news"
  "Links": [
    {
      "Id": 11,
      "Text": "News",
      "Link": "/news"
    },
    {
      "Id": 21,
      "Text": "More News",
      "Link": "/morenews"
    }
  ]
}

Classes

public class Pages
{
    public string Title { get; set; }
    public string Head { get; set; }
    public List<Links> Links { get; set; }
}

public class Links
{
    public int Id { get; set; }
    public string Text { get; set; }
    public string Link { get; set; }
}

C# code:

public async Task PopulateObjectAsync(object target, string source, Type type, bool replaceArrays = false)
{
    using var json = JsonDocument.Parse(source).RootElement;

    if (json.ValueKind.Equals(JsonValueKind.Array))
    {
        foreach (var item in json.EnumerateArray())
        {
            foreach (var property in item.EnumerateObject())
            {
                await OverwriteProperty(???, replaceArray);  //use "replaceArray" here ?
            }
        }
    }
    else
    {
        foreach (var property in json.EnumerateObject())
        {
            await OverwriteProperty(target, property, type, replaceArray);  //use "replaceArray" here ?
        }
    }

    return;
}

public async Task OverwriteProperty(object target, JsonProperty updatedProperty, Type type, bool replaceArrays)
{
    var propertyInfo = type.GetProperty(updatedProperty.Name);

    if (propertyInfo == null)
    {
        return;
    }

    var propertyType = propertyInfo.PropertyType;
    object parsedValue;

    if (propertyType.IsValueType)
    {
        parsedValue = JsonSerializer.Deserialize(
            updatedProperty.Value.GetRawText(),
            propertyType);
    }
    else if (replaceArrays && "property is JsonValueKind.Array")  //pseudo code sample
    {
        // use same code here as in above "IsValueType" ?
    }
    else
    {
        parsedValue = propertyInfo.GetValue(target);

        await PopulateObjectAsync(
            parsedValue,
            updatedProperty.Value.GetRawText(),
            propertyType);
    }

    propertyInfo.SetValue(target, parsedValue);
}

Answer 1

Well, If you don't care how the arrays are written, I have a simple solution. Create a new JSON within 2 phases 1 loop for new properties and 1 loop for the updates:

    var sourceJson = @"
{
  ""Title"": ""Startpage"",
  ""Links"": [
    {
      ""Id"": 10,
      ""Text"": ""Start"",
      ""Link"": ""/index""
    },
    {
      ""Id"": 11,
      ""Text"": ""Info"",
      ""Link"": ""/info""
    }
  ]
}";
        var updateJson = @"
{
  ""Head"": ""Latest news"",
  ""Links"": [
    {
      ""Id"": 11,
      ""Text"": ""News"",
      ""Link"": ""/news""
    },
    {
      ""Id"": 21,
      ""Text"": ""More News"",
      ""Link"": ""/morenews""
    }
  ]
}
";
        using var source = JsonDocument.Parse(sourceJson);
        using var update = JsonDocument.Parse(updateJson);
        using var stream = new MemoryStream();
        using var writer = new Utf8JsonWriter(stream);
        writer.WriteStartObject();
        // write non existing properties
        foreach (var prop in update.RootElement.EnumerateObject().Where(prop => !source.RootElement.TryGetProperty(prop.Name, out _)))
        {
            prop.WriteTo(writer);
        }

        // make updates for existing
        foreach (var prop in source.RootElement.EnumerateObject())
        {
            if (update.RootElement.TryGetProperty(prop.Name, out var overwrite))
            {
                writer.WritePropertyName(prop.Name);
                overwrite.WriteTo(writer);
            }
            else
            {
                prop.WriteTo(writer);
            }
        }

        writer.WriteEndObject();
        writer.Flush();
        var resultJson = Encoding.UTF8.GetString(stream.ToArray());
        Console.WriteLine(resultJson);

Output :

{
   "Head":"Latest news",
   "Title":"Startpage",
   "Links":[
      {
         "Id":11,
         "Text":"News",
         "Link":"/news"
      },
      {
         "Id":21,
         "Text":"More News",
         "Link":"/morenews"
      }
   ]
}

Fiddle

Answer 2

Preliminaries

I'll be heavily working with the existing code from my answer to the linked question: .Net Core 3.0 JsonSerializer populate existing object.

As I mentioned, the code for shallow copies works and produces Result 2. So we only need to fix the code for deep copying and get it to produce Result 1.

On my machine the code crashes in PopulateObject when the propertyType is typeof(string), since string is neither a value type nor something represented by an object in JSON. I fixed that back in the original answer, the if must be:

if (elementType.IsValueType || elementType == typeof(string))

Implementing the new requirements

Okay, so the first issue is recognising whether something is a collection. Currently we look at the type of the property that we want to overwrite to make a decision, so now we will do the same. The logic is as follows:

private static bool IsCollection(Type type) =>
        type.GetInterfaces().Any(x => x.IsGenericType && 
        x.GetGenericTypeDefinition() == typeof(ICollection<>));

So the only things we consider collections are things that implement ICollection<T> for some T. We will handle collections completely separately by implementing a new PopulateCollection method. We will also need a way to construct a new collection - maybe the list in the initial object is null, so we need to create a new one before populating it. For that we'll look for its parameterless constructor:

private static object Instantiate(Type type)
{
    var ctor =  type.GetConstructor(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic, Array.Empty<Type>());

    if (ctor is null)
    {
        throw new InvalidOperationException($"Type {type.Name} has no parameterless constructor.");
    }

    return ctor.Invoke(Array.Empty<object?>());
}

We allow it to be private, because why not.

Now we make some changes to OverwriteProperty:

    private static void OverwriteProperty(object target, JsonProperty updatedProperty, Type type)
    {
        var propertyInfo = type.GetProperty(updatedProperty.Name);

        if (propertyInfo == null)
        {
            return;
        }

        if (updatedProperty.Value.ValueKind == JsonValueKind.Null)
        {
            propertyInfo.SetValue(target, null);
            return;
        }

        var propertyType = propertyInfo.PropertyType;
        object? parsedValue;

        if (propertyType.IsValueType || propertyType == typeof(string))
        {
            parsedValue = JsonSerializer.Deserialize(
                updatedProperty.Value.GetRawText(),
                propertyType);
        }
        else if (IsCollection(propertyType))
        {
            var elementType = propertyType.GenericTypeArguments[0];
            parsedValue = propertyInfo.GetValue(target);
            parsedValue ??= Instantiate(propertyType);

            PopulateCollection(parsedValue, updatedProperty.Value.GetRawText(), elementType);
        }
        else
        {
            parsedValue = propertyInfo.GetValue(target);
            parsedValue ??= Instantiate(propertyType);

            PopulateObject(
                parsedValue,
                updatedProperty.Value.GetRawText(),
                propertyType);
        }

        propertyInfo.SetValue(target, parsedValue);
    }

The big change is the second branch of the if statement. We find out the type of the elements in the collection and extract the existing collection from the object. If it is null, we create a new, empty one. Then we call the new method to populate it.

The PopulateCollection method will be very similar to OverwriteProperty.

private static void PopulateCollection(object target, string jsonSource, Type elementType)

First we get the Add method of the collection:

var addMethod = target.GetType().GetMethod("Add", new[] { elementType });

Here we expect an actual JSON array, so it's time to enumerate it. For every element in the array we need to do the same thing as in OverwriteProperty, depending on whether we have a value, array or object we have different flows.

foreach (var property in json.EnumerateArray())
{
    object? element;

    if (elementType.IsValueType || elementType == typeof(string))
    {
        element = JsonSerializer.Deserialize(jsonSource, elementType);
    }
    else if (IsCollection(elementType))
    {
        var nestedElementType = elementType.GenericTypeArguments[0];
        element = Instantiate(elementType);

        PopulateCollection(element, property.GetRawText(), nestedElementType);
    }
    else
    {
        element = Instantiate(elementType);

        PopulateObject(element, property.GetRawText(), elementType);
    }

    addMethod.Invoke(target, new[] { element });
}

Uniqueness

Now we have an issue. The current implementation will always add to the collection, regardless of its current contents. So the thing this would return is neither Result 1 nor Result 2, it'd be Result 3:

{
  "Title": "Startpage",
  "Head": "Latest news"
  "Links": [
    {
      "Id": 10,
      "Text": "Start",
      "Link": "/indexnews"
    },
    {
      "Id": 11,
      "Text": "News",
      "Link": "/news"
    },
    {
      "Id": 11,
      "Text": "News",
      "Link": "/news"
    },
    {
      "Id": 21,
      "Text": "More news",
      "Link": "/morenews"
    }
  ]
}

We had the array with links 10 and 11 and then added another one with links 11 and 12. There is no obvious natural way of dealing with this. The design decision I chose here is: the collection decides whether the element is already there. We will call the default Contains method on the collection and add if and only if it returns false. It requires us to override the Equals method on Links to compare the Id:

public override bool Equals(object? obj) =>
    obj is Links other && Id == other.Id;

public override int GetHashCode() => Id.GetHashCode();

Now the changes required are:

• First, fetch the Contains method:

var containsMethod = target.GetType().GetMethod("Contains", new[] { elementType });

• Then, check it after we get an element:

var contains = containsMethod.Invoke(target, new[] { element });
if (contains is false)
{
    addMethod.Invoke(target, new[] { element });
}

Tests

I add a few things to your Pages and Links class, first of all I override ToString so we can easily check our results. Then, as mentioned, I override Equals for Links:

public class Pages
{
    public string Title { get; set; }
    public string Head { get; set; }
    public List<Links> Links { get; set; }

    public override string ToString() => 
        $"Pages {{ Title = {Title}, Head = {Head}, Links = {string.Join(", ", Links)} }}";
}

public class Links
{
    public int Id { get; set; }
    public string Text { get; set; }
    public string Link { get; set; }

    public override bool Equals(object? obj) =>
        obj is Links other && Id == other.Id;

    public override int GetHashCode() => Id.GetHashCode();

    public override string ToString() => $"Links {{ Id = {Id}, Text = {Text}, Link = {Link} }}";
}

And the test:

var initial = @"{
  ""Title"": ""Startpage"",
  ""Links"": [
    {
      ""Id"": 10,
      ""Text"": ""Start"",
      ""Link"": ""/index""
    },
    {
    ""Id"": 11,
      ""Text"": ""Info"",
      ""Link"": ""/info""
    }
  ]
}";

var update = @"{
  ""Head"": ""Latest news"",
  ""Links"": [
    {
      ""Id"": 11,
      ""Text"": ""News"",
      ""Link"": ""/news""
    },
    {
    ""Id"": 21,
      ""Text"": ""More News"",
      ""Link"": ""/morenews""
    }
  ]
}";

var pages = new Pages();

PopulateObject(pages, initial);

Console.WriteLine(pages);

PopulateObject(pages, update);

Console.WriteLine(pages);

The result:

Initial:
Pages { Title = Startpage, Head = , Links = Links { Id = 10, Text = Start, Link = /index }, Links { Id = 11, Text = Info, Link = /info } }
Update:
Pages { Title = Startpage, Head = Latest news, Links = Links { Id = 10, Text = Start, Link = /index }, Links { Id = 11, Text = Info, Link = /info }, Links { Id = 21, Text = More News, Link = /morenews } }

You can find it in this fiddle.

Limitations

1. We use the Add method, so this will not work on properties that are .NET arrays, since you can't Add to them. They would have to be handled separately, where you first create the elements, then construct an array of an appropriate size and fill it.
2. The decision to use Contains is a bit iffy to me. It would be nice to have better control on what gets added to the collection. But this is simple and works, so it will be enough for an SO answer.

Final code

static class JsonUtils
{
    public static void PopulateObject<T>(T target, string jsonSource) where T : class =>
        PopulateObject(target, jsonSource, typeof(T));

    public static void OverwriteProperty<T>(T target, JsonProperty updatedProperty) where T : class =>
        OverwriteProperty(target, updatedProperty, typeof(T));

    private static void PopulateObject(object target, string jsonSource, Type type)
    {
        using var json = JsonDocument.Parse(jsonSource).RootElement;

        foreach (var property in json.EnumerateObject())
        {
            OverwriteProperty(target, property, type);
        }
    }

    private static void PopulateCollection(object target, string jsonSource, Type elementType)
    {
        using var json = JsonDocument.Parse(jsonSource).RootElement;
        var addMethod = target.GetType().GetMethod("Add", new[] { elementType });
        var containsMethod = target.GetType().GetMethod("Contains", new[] { elementType });

        Debug.Assert(addMethod is not null);
        Debug.Assert(containsMethod is not null);

        foreach (var property in json.EnumerateArray())
        {
            object? element;

            if (elementType.IsValueType || elementType == typeof(string))
            {
                element = JsonSerializer.Deserialize(jsonSource, elementType);
            }
            else if (IsCollection(elementType))
            {
                var nestedElementType = elementType.GenericTypeArguments[0];
                element = Instantiate(elementType);

                PopulateCollection(element, property.GetRawText(), nestedElementType);
            }
            else
            {
                element = Instantiate(elementType);

                PopulateObject(element, property.GetRawText(), elementType);
            }

            var contains = containsMethod.Invoke(target, new[] { element });
            if (contains is false)
            {
                addMethod.Invoke(target, new[] { element });
            }
        }
    }

    private static void OverwriteProperty(object target, JsonProperty updatedProperty, Type type)
    {
        var propertyInfo = type.GetProperty(updatedProperty.Name);

        if (propertyInfo == null)
        {
            return;
        }

        if (updatedProperty.Value.ValueKind == JsonValueKind.Null)
        {
            propertyInfo.SetValue(target, null);
            return;
        }

        var propertyType = propertyInfo.PropertyType;
        object? parsedValue;

        if (propertyType.IsValueType || propertyType == typeof(string))
        {
            parsedValue = JsonSerializer.Deserialize(
                updatedProperty.Value.GetRawText(),
                propertyType);
        }
        else if (IsCollection(propertyType))
        {
            var elementType = propertyType.GenericTypeArguments[0];
            parsedValue = propertyInfo.GetValue(target);
            parsedValue ??= Instantiate(propertyType);

            PopulateCollection(parsedValue, updatedProperty.Value.GetRawText(), elementType);
        }
        else
        {
            parsedValue = propertyInfo.GetValue(target);
            parsedValue ??= Instantiate(propertyType);

            PopulateObject(
                parsedValue,
                updatedProperty.Value.GetRawText(),
                propertyType);
        }

        propertyInfo.SetValue(target, parsedValue);
    }

    private static object Instantiate(Type type)
    {
        var ctor =  type.GetConstructor(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic, Array.Empty<Type>());

        if (ctor is null)
        {
            throw new InvalidOperationException($"Type {type.Name} has no parameterless constructor.");
        }

        return ctor.Invoke(Array.Empty<object?>());
    }

    private static bool IsCollection(Type type) =>
        type.GetInterfaces().Any(x => x.IsGenericType && x.GetGenericTypeDefinition() == typeof(ICollection<>));
}

Answer 3

After further consideration, I think a simpler solution for replacement should be using C# Reflection instead of relying on JSON. Tell me if it does not satisfy your need:

public class JsonPopulator
{


    public static void PopulateObjectByReflection(object target, string json, bool replaceArray)
    {
        var type = target.GetType();
        var replacements = JsonSerializer.Deserialize(json, type);

        PopulateSubObject(target, replacements, replaceArray);
    }

    static void PopulateSubObject(object target, object? replacements, bool replaceArray)
    {
        if (replacements == null) { return; }

        var props = target.GetType().GetProperties();

        foreach (var prop in props)
        {
            // Skip if can't write
            if (!prop.CanWrite) { continue; }

            // Skip if no value in replacement
            var propType = prop.PropertyType;
            var replaceValue = prop.GetValue(replacements);
            if (replaceValue == GetDefaultValue(propType)) { continue; }

            // Now check if it's array AND we do not want to replace it            
            if (replaceValue is IEnumerable<object> replacementList)
            {
                var currList = prop.GetValue(target) as IEnumerable<object>;

                
                var finalList = replaceValue;
                // If there is no initial list, or if we simply want to replace the array
                if (currList == null || replaceArray)
                {
                    // Do nothing here, we simply replace it
                }
                else
                {
                    // Append items at the end
                    finalList = currList.Concat(replacementList);

                    // Since casting logic is complicated, we use a trick to just
                    // Serialize then Deserialize it again
                    // At the cost of performance hit if it's too big
                    var listJson = JsonSerializer.Serialize(finalList);
                    finalList = JsonSerializer.Deserialize(listJson, propType);
                }

                prop.SetValue(target, finalList);
            }
            else if (propType.IsValueType || propType == typeof(string))
            {
                // Simply copy value over
                prop.SetValue(target, replaceValue);
            }
            else
            {
                // Recursively copy child properties
                var subTarget = prop.GetValue(target);
                var subReplacement = prop.GetValue(replacements);

                // Special case: if original object doesn't have the value
                if (subTarget == null && subReplacement != null)
                {
                    prop.SetValue(target, subReplacement);
                }
                else
                {
                    PopulateSubObject(target, replacements, replaceArray);
                }
            }
        }
    }

    // From https://stackoverflow.com/questions/325426/programmatic-equivalent-of-defaulttype
    static object? GetDefaultValue(Type type)
    {
        if (type.IsValueType)
        {
            return Activator.CreateInstance(type);
        }
        return null;
    }
}

Using:

const string Json1 = "{\n  \"Title\": \"Startpage\",\n  \"Links\": [\n    {\n      \"Id\": 10,\n      \"Text\": \"Start\",\n      \"Link\": \"/index\"\n    },\n    {\n      \"Id\": 11,\n      \"Text\": \"Info\",\n      \"Link\": \"/info\"\n    }\n  ]\n}";

const string Json2 = "{\n  \"Head\": \"Latest news\",\n  \"Links\": [\n    {\n      \"Id\": 11,\n      \"Text\": \"News\",\n      \"Link\": \"/news\"\n    },\n    {\n      \"Id\": 21,\n      \"Text\": \"More News\",\n      \"Link\": \"/morenews\"\n    }\n  ]\n}";

var obj = JsonSerializer.Deserialize<Pages>(Json1)!;

JsonPopulator.PopulateObjectByReflection(obj, Json2, false);
Console.WriteLine(obj.Links.Count); // 4

JsonPopulator.PopulateObjectByReflection(obj, Json2, true);
Console.WriteLine(obj.Links.Count); // 2

The solution even works when I replace List<Links> with array Links[]:

public class Pages
{
    // ...
    public Links[] Links { get; set; }
}

JsonPopulator.PopulateObjectByReflection(obj, Json2, false);
Console.WriteLine(obj.Links.Length); // 4

JsonPopulator.PopulateObjectByReflection(obj, Json2, true);
Console.WriteLine(obj.Links.Length); // 2

---

Abandoned solution:

I think a simple solution would be to include the parent and its current property info. One reason is that not every IEnumerable is mutable anyway (Array for example) so you will want to replace it even with replaceArray being false.

using System.Collections;
using System.Diagnostics.CodeAnalysis;
using System.Reflection;
using System.Text.Json;

const string Json1 = @"
    {
        ""Bars"": [
            { ""Value"": 0 },
            { ""Value"": 1 }
        ]
    }
";

const string Json2 = @"
    {
        ""Bars"": [
            { ""Value"": 2 },
            { ""Value"": 3 }
        ]
    }
";

var foo = JsonSerializer.Deserialize<Foo>(Json1)!;

PopulateObject(foo, Json2, false);
Console.WriteLine(foo.Bars.Count); // 4

PopulateObject(foo, Json2, true);
Console.WriteLine(foo.Bars.Count); // 2

static void PopulateObject(object target, string replacement, bool replaceArray)
{

    using var doc = JsonDocument.Parse(Json2);
    var root = doc.RootElement;

    PopulateObjectWithJson(target, root, replaceArray, null, null);
}

static void PopulateObjectWithJson(object target, JsonElement el, bool replaceArray, object? parent, PropertyInfo? parentProp)
{
    // There should be other checks
    switch (el.ValueKind)
    {
        case JsonValueKind.Object:
            // Just simple check here, you may want more logic
            var props = target.GetType().GetProperties().ToDictionary(q => q.Name);

            foreach (var jsonProp in el.EnumerateObject())
            {
                if (props.TryGetValue(jsonProp.Name, out var prop))
                {
                    var subTarget = prop.GetValue(target);

                    // You may need to check for null etc here
                    ArgumentNullException.ThrowIfNull(subTarget);

                    PopulateObjectWithJson(subTarget, jsonProp.Value, replaceArray, target, prop);
                }
            }

            break;
        case JsonValueKind.Array:
            var parsedItems = new List<object>();
            foreach (var item in el.EnumerateArray())
            {
                // Parse your value here, I will just assume the type for simplicity
                var bar = new Bar()
                {
                    Value = item.GetProperty(nameof(Bar.Value)).GetInt32(),
                };

                parsedItems.Add(bar);
            }

            IEnumerable<object> finalItems = parsedItems;
            if (!replaceArray)
            {
                finalItems = ((IEnumerable<object>)target).Concat(parsedItems);
            }

            // Parse your list into List/Array/Collection/etc
            // You need reflection here as well
            var list = finalItems.Cast<Bar>().ToList();
            parentProp?.SetValue(parent, list);

            break;
        default:
            // Should handle for other types
            throw new NotImplementedException();
    }
}

public class Foo
{

    public List<Bar> Bars { get; set; } = null!;

}

public class Bar
{
    public int Value { get; set; }
}