In the process of upgrading to ASP.NET Core 5, we have encountered a situation where we need to serialize and return a Json.NET JObject (returned by some legacy code we can't yet change) using System.Text.Json. How can this be done in a reasonably efficient manner, without re-serializing and re-parsing the JSON to a JsonDocument or reverting back to Json.NET completely via AddNewtonsoftJson()?
Specifically, say we have the following legacy data model:
public class Model
{
public JObject Data { get; set; }
}
When we return this from ASP.NET Core 5.0, the contents of the "value" property get mangled into a series of empty arrays. E.g.:
var inputJson = @"{""value"":[[null,true,false,1010101,1010101.10101,""hello"",""𩸽"",""\uD867\uDE3D"",""2009-02-15T00:00:00Z"",""\uD867\uDE3D\u0022\\/\b\f\n\r\t\u0121""]]}";
var model = new Model { Data = JObject.Parse(inputJson) };
var outputJson = JsonSerializer.Serialize(model);
Console.WriteLine(outputJson);
Assert.IsTrue(JToken.DeepEquals(JToken.Parse(inputJson), JToken.Parse(outputJson)[nameof(Model.Data)]));
Fails, and generates the following incorrect JSON:
{"Data":{"value":[[[],[],[],[],[],[],[],[],[],[]]]}}
How can I correctly serialize the JObject property with System.Text.Json? Note that the JObject can be fairly large so we would prefer to stream it out rather than format it to a string and parse it again from scratch into a JsonDocument simply to return it.
It is necessary to create a custom JsonConverterFactory to serialize a Json.NET JToken hierarchy to JSON using System.Text.Json.
Since the question seeks to avoid re-serializing the entire JObject to JSON just to parse it again using System.Text.Json, the following converter descends the token hierarchy recursively writing each individual value out to the Utf8JsonWriter:
using System.Text.Json;
using System.Text.Json.Serialization;
using Newtonsoft.Json.Linq;
public class JTokenConverterFactory : JsonConverterFactory
{
// Cache well known converters to avoid problems in Native AOT mode
static readonly IReadOnlyDictionary<Type, Func<JsonSerializerOptions, Newtonsoft.Json.JsonSerializerSettings?, JsonConverter>> WellKnownConverterFactories =
new Dictionary<Type, Func<JsonSerializerOptions, Newtonsoft.Json.JsonSerializerSettings?, JsonConverter>>()
{
[typeof(JToken)] = (options, settings) => new JTokenConverter<JToken>(options, settings),
[typeof(JValue)] = (options, settings) => new JTokenConverter<JValue>(options, settings),
[typeof(JRaw)] = (options, settings) => new JTokenConverter<JRaw>(options, settings),
[typeof(JContainer)] = (options, settings) => new JTokenConverter<JContainer>(options, settings),
[typeof(JObject)] = (options, settings) => new JTokenConverter<JObject>(options, settings),
[typeof(JArray)] = (options, settings) => new JTokenConverter<JArray>(options, settings),
[typeof(JConstructor)] = (options, settings) => throw new JsonException("Serialization of non-standard JConstructor token with System.Text.Json is not supported."),
[typeof(JProperty)] = (options, settings) => throw new JsonException("JProperty cannot be serialized from or to JSON as a standalone object."),
};
// In case you need to set FloatParseHandling or DateFormatHandling
readonly Newtonsoft.Json.JsonSerializerSettings? settings;
public JTokenConverterFactory(Newtonsoft.Json.JsonSerializerSettings? settings) => this.settings = settings;
public JTokenConverterFactory() : this(null) { }
public override bool CanConvert(Type typeToConvert) => typeof(JToken).IsAssignableFrom(typeToConvert);
public override JsonConverter CreateConverter(Type typeToConvert, JsonSerializerOptions options)
{
if (WellKnownConverterFactories.TryGetValue(typeToConvert, out var factory))
return factory(options, settings);
// All known JToken types (as of Json.NET 13.0) are included in WellKnownConverterFactories, the below call to MakeGenericType() is future-proofing.
// So, in native AOT mode, one could remove the MakeGenericType() call and just do:
// throw new JsonException($"Unknown JToken type {typeToConvert}");
var converterType = typeof(JTokenConverter<>).MakeGenericType(new [] { typeToConvert} );
return (JsonConverter)Activator.CreateInstance(converterType, new object? [] { options, settings } )!;
}
class JTokenConverter<TJToken> : JsonConverter<TJToken> where TJToken : JToken
{
readonly Newtonsoft.Json.JsonSerializerSettings? settings;
public JTokenConverter(JsonSerializerOptions options, Newtonsoft.Json.JsonSerializerSettings? settings) => this.settings = settings;
public override bool CanConvert(Type typeToConvert) => typeof(TJToken).IsAssignableFrom(typeToConvert);
public override TJToken? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
using var newtonsoftWriter = new JTokenWriter();
ReadCore(ref reader, newtonsoftWriter, options, typeof(TJToken), Newtonsoft.Json.JsonSerializer.CreateDefault(settings));
return (TJToken?)newtonsoftWriter.Token;
}
public override void Write(Utf8JsonWriter writer, TJToken value, JsonSerializerOptions options) =>
// Optimize for memory use by descending the JToken hierarchy and writing each one out, rather than formatting to a string, parsing to a `JsonDocument`, then writing that.
WriteCore(writer, value, options);
static void WriteCore(Utf8JsonWriter writer, JToken value, JsonSerializerOptions options)
{
if (value == null || value.Type == JTokenType.Null)
{
writer.WriteNullValue();
return;
}
switch (value)
{
case JValue jvalue when jvalue.GetType() != typeof(JValue): // JRaw, maybe others
default: // etc
{
// We could just format the JToken to a string, but System.Text.Json works more efficiently with UTF8 byte streams so write to one of those instead.
using var ms = new MemoryStream();
using (var tw = new StreamWriter(ms, leaveOpen : true))
using (var jw = new Newtonsoft.Json.JsonTextWriter(tw))
{
value.WriteTo(jw);
}
ms.Position = 0;
using var doc = JsonDocument.Parse(ms);
doc.WriteTo(writer);
}
break;
// Hardcode some standard cases for efficiency
case JValue jvalue when jvalue.Value is null:
writer.WriteNullValue();
break;
case JValue jvalue when jvalue.Value is bool v:
writer.WriteBooleanValue(v);
break;
case JValue jvalue when jvalue.Value is string v:
writer.WriteStringValue(v);
break;
case JValue jvalue when jvalue.Value is long v:
writer.WriteNumberValue(v);
break;
case JValue jvalue when jvalue.Value is int v:
writer.WriteNumberValue(v);
break;
case JValue jvalue when jvalue.Value is decimal v:
writer.WriteNumberValue(v);
break;
case JValue jvalue when jvalue.Value is double v:
writer.WriteNumberValue(v);
break;
case JValue jvalue:
JsonSerializer.Serialize(writer, jvalue.Value, options);
break;
case JArray array:
{
writer.WriteStartArray();
foreach (var item in array)
WriteCore(writer, item, options);
writer.WriteEndArray();
}
break;
case JObject obj:
{
writer.WriteStartObject();
foreach (var p in obj.Properties())
{
writer.WritePropertyName(p.Name);
WriteCore(writer, p.Value, options);
}
writer.WriteEndObject();
}
break;
}
}
static void ReadArrayCore(ref Utf8JsonReader reader, Newtonsoft.Json.JsonWriter newtonsoftWriter, JsonSerializerOptions options, Newtonsoft.Json.JsonSerializer newtonsoftSerializer)
{
if (reader.TokenType != JsonTokenType.StartArray)
throw new JsonException();
newtonsoftWriter.WriteStartArray();
while (reader.Read())
{
switch (reader.TokenType)
{
default:
ReadCore(ref reader, newtonsoftWriter, options, typeof(JToken), newtonsoftSerializer);
break;
case JsonTokenType.EndArray:
newtonsoftWriter.WriteEndArray();
return;
}
}
throw new JsonException();
}
static void ReadObjectCore(ref Utf8JsonReader reader, Newtonsoft.Json.JsonWriter newtonsoftWriter, JsonSerializerOptions options, Newtonsoft.Json.JsonSerializer newtonsoftSerializer)
{
if (reader.TokenType != JsonTokenType.StartObject)
throw new JsonException();
newtonsoftWriter.WriteStartObject();
while (reader.Read())
{
switch (reader.TokenType)
{
case JsonTokenType.PropertyName:
var name = reader.GetString()!;
if (!reader.Read())
throw new JsonException();
newtonsoftWriter.WritePropertyName(name);
ReadCore(ref reader, newtonsoftWriter, options, typeof(JToken), newtonsoftSerializer);
break;
case JsonTokenType.EndObject:
newtonsoftWriter.WriteEndObject();
return;
default:
throw new JsonException();
}
}
throw new JsonException();
}
static void CheckType(Type extectedType, Type actualType, JsonTokenType tokenType)
{
if (!extectedType.IsAssignableFrom(actualType))
throw new JsonException(string.Format("Expected type {0} cannot be created from token {1}", extectedType, tokenType));
}
static void ReadCore(ref Utf8JsonReader reader, Newtonsoft.Json.JsonWriter newtonsoftWriter, JsonSerializerOptions options, Type extectedType, Newtonsoft.Json.JsonSerializer newtonsoftSerializer)
{
var t = reader.TokenType;
switch (reader.TokenType)
{
case JsonTokenType.Comment:
CheckType(extectedType, typeof(JValue), reader.TokenType);
newtonsoftWriter.WriteComment(reader.GetString());
break;
case JsonTokenType.False:
CheckType(extectedType, typeof(JValue), reader.TokenType);
newtonsoftWriter.WriteValue(false);
break;
case JsonTokenType.True:
CheckType(extectedType, typeof(JValue), reader.TokenType);
newtonsoftWriter.WriteValue(true);
break;
case JsonTokenType.Null:
CheckType(extectedType, typeof(JValue), reader.TokenType);
newtonsoftWriter.WriteNull();
break;
case JsonTokenType.String:
{
// To ensure that DateTime values are recognized consistently with Json.NET conventions, we must invoke the Json.NET
// serializer or parser using the incoming JsonSerializerSettings. See https://www.newtonsoft.com/json/help/html/DatesInJSON.htm
CheckType(extectedType, typeof(JValue), reader.TokenType);
using var newtonsoftReader = new Newtonsoft.Json.JsonTextReader(new StringReader(Newtonsoft.Json.JsonConvert.ToString(reader.GetString())));
newtonsoftSerializer.Deserialize<JValue>(newtonsoftReader)!.WriteTo(newtonsoftWriter);
}
break;
case JsonTokenType.Number:
CheckType(extectedType, typeof(JValue), reader.TokenType);
// For efficiency, see if the value is an integer convertible to long.
if (reader.TryGetInt64(out var l))
newtonsoftWriter.WriteValue(l);
else
{
// Inconsistencies in numeric formats cause difficulties here. System.Text.Json keeps the underlying byte representation while JsonReader parses
// to the closest fit .Net numeric type (long, double, decimal or BigInteger). Just get the raw JSON and let Newtonsoft handle the conversion.
using var doc = JsonDocument.ParseValue(ref reader);
using var newtonsoftReader = new Newtonsoft.Json.JsonTextReader(new StringReader(doc.RootElement.ToString()));
newtonsoftSerializer.Deserialize<JValue>(newtonsoftReader)!.WriteTo(newtonsoftWriter);
}
break;
case JsonTokenType.StartArray:
CheckType(extectedType, typeof(JArray), reader.TokenType);
ReadArrayCore(ref reader, newtonsoftWriter, options, newtonsoftSerializer);
break;
case JsonTokenType.StartObject:
CheckType(extectedType, typeof(JObject), reader.TokenType);
ReadObjectCore(ref reader, newtonsoftWriter, options, newtonsoftSerializer);
break;
}
}
}
}
Then the unit test in the question should be modified to use the following JsonSerializerOptions:
var options = new JsonSerializerOptions
{
Converters = { new JTokenConverterFactory() },
};
var outputJson = JsonSerializer.Serialize(model, options);
Notes:
The converter implements both streaming serialization and deserialization of JToken types.
Newtonsoft's JsonSerializerSettings may be passed to customize settings such as FloatParseHandling or DateFormatHandling during deserialization.
To add JTokenConverterFactory to the ASP.NET Core serialization options, see Configure System.Text.Json-based formatters.
Demo fiddle with some basic tests here: https://dotnetfiddle.net/KFJjH4.