Skip to content

Commit 9419d16

Browse files
NikolaMilosavljevicNikolaMilosavljevic
authored
Newtonsoft.Json → System.Text.Json Migration (#9956)
# Summary of changes ## 1. Core Shared Infrastructure (1 file) - **JExtensions.cs** (`src/Shared/JExtensions.cs`) — **Most significant change.** This shared file (linked into multiple projects) was the central Newtonsoft.Json helper. Completely rewritten to use `System.Text.Json.Nodes` (`JsonNode`, `JsonObject`, `JsonArray`, `JsonValue`) instead of `JObject`/`JToken`/`JArray`. Key changes: - All extension methods now operate on `JsonObject`/`JsonNode`/`JsonArray` instead of Newtonsoft types - `ParseJsonNode()` / `ParseJsonObject()` helpers with `JsonDocumentOptions` configured for comment handling and trailing commas - `ToJsonString()` for serialization using `JsonSerializer` - String/int/bool/enum/array/object accessor methods (`ToString()`, `ToInt32()`, `Get<T>()`, etc.) rewritten for STJ APIs - `JsonNodeComparer` and `JsonNodeEqualityComparer` utility classes added ## 2. Package References (5 files) - **Directory.Packages.props** — Removed `Newtonsoft.Json` package reference; added `System.Text.Json` and `JsonSchema.Net` (for JSON schema validation in tests). - 4 csproj files — Replaced `Newtonsoft.Json` with `System.Text.Json`, conditioned with `Condition="'$(TargetFrameworkIdentifier)' != '.NETCoreApp'"` to avoid NU1510 package pruning warnings (System.Text.Json is inbox for .NET 10+, but still needed for `net472`/`netstandard2.0`): - `src/Microsoft.TemplateEngine.Edge/Microsoft.TemplateEngine.Edge.csproj` - `src/Microsoft.TemplateEngine.Orchestrator.RunnableProjects/Microsoft.TemplateEngine.Orchestrator.RunnableProjects.csproj` - `src/Microsoft.TemplateEngine.Utils/Microsoft.TemplateEngine.Utils.csproj` - `src/Microsoft.TemplateSearch.Common/Microsoft.TemplateSearch.Common.csproj` - **Microsoft.TemplateSearch.TemplateDiscovery.csproj** — Removed `Newtonsoft.Json` reference. - **RunnableProjects.UnitTests.csproj** — Replaced `Newtonsoft.Json` with `JsonSchema.Net` for schema validation tests. ## 3. Production Source Files (62 files) ### Microsoft.TemplateEngine.Edge (6 files) - All converted from `JObject`/`JToken` to `JsonObject`/`JsonNode` ### Microsoft.TemplateEngine.Orchestrator.RunnableProjects (32 files) ### Microsoft.TemplateSearch.Common (11 files) ### Tools/TemplateDiscovery (12 files) ### JsonEncodeValueFormFactory.cs Added `JavaScriptEncoder.UnsafeRelaxedJsonEscaping` so `"` encodes as `\"` (matching Newtonsoft behavior) instead of STJ's default `\u0022`. ## 4. Test Files (24 files) All test files updated to use `System.Text.Json` APIs instead of Newtonsoft: ## 5. Test Data Fixes (4 files) Removed duplicate `$schema` properties from template.json files that Newtonsoft silently tolerated but STJ/JsonSchema.Net rejects: - `test_templates/TemplateConditionalProcessing/.template.config/template.json` - `test_templates/Invalid/InvalidHostData/.template.config/template.json` - `test_templates/SourceWithExcludeAndWithout/With/.template.config/template.json` - `test_templates/SourceWithExcludeAndWithout/Without/.template.config/template.json` ## 6. Behavioral Differences Addressed | Issue | Newtonsoft Behavior | STJ Behavior | Fix Applied | |-------|-------------------|--------------|-------------| | Single-quoted JSON | Allowed | Rejected | Fixed test data to use double quotes | | Unquoted property names | Allowed | Rejected | Fixed test data to use quoted names | | Boolean serialization | `"True"` / `"False"` | `"true"` / `"false"` | Updated test assertions | | `"` encoding in JSON strings | `\"` | `\u0022` | Added `UnsafeRelaxedJsonEscaping` | | Duplicate JSON keys | Last value wins | Throws `ArgumentException` | Removed duplicates from test data | | `JsonNode.ToString()` format | Indented | Indented (but `ToJsonString()` is compact) | Used `ToJsonString()` in assertions | ## 7. Validation [VMR build](https://dev.azure.com/dnceng/internal/_build/results?buildId=2922641&view=results)
2 parents 0756f96 + 84da25e commit 9419d16

113 files changed

Lines changed: 1174 additions & 1083 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Directory.Packages.props‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,7 @@
4040
<ItemGroup>
4141
<PackageVersion Include="FakeItEasy" Version="8.1.0" />
4242
<PackageVersion Include="AwesomeAssertions" Version="8.0.2" />
43-
<PackageVersion Include="Newtonsoft.Json.Schema" Version="4.0.1" />
44-
<PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
43+
<PackageVersion Include="JsonSchema.Net" Version="7.3.2" />
4544
<PackageVersion Include="StyleCop.Analyzers" Version="1.2.0-beta.556" />
4645
<PackageVersion Include="Verify.DiffPlex" Version="3.0.0" />
4746
<PackageVersion Include="Verify.XUnit" Version="25.0.2" />
@@ -62,7 +61,7 @@
6261
<PackageVersion Update="Microsoft.Extensions.Logging.Abstractions" Version="$(MicrosoftExtensionsLoggingAbstractionsVersion)" Condition="'$(MicrosoftExtensionsLoggingAbstractionsVersion)' != ''" />
6362
<PackageVersion Update="Microsoft.Extensions.Logging.Console" Version="$(MicrosoftExtensionsLoggingConsoleVersion)" Condition="'$(MicrosoftExtensionsLoggingConsoleVersion)' != ''" />
6463
<PackageVersion Update="Microsoft.Extensions.Logging" Version="$(MicrosoftExtensionsLoggingVersion)" Condition="'$(MicrosoftExtensionsLoggingVersion)' != ''" />
65-
<PackageVersion Update="Newtonsoft.Json" Version="$(NewtonsoftJsonVersion)" Condition="'$(NewtonsoftJsonVersion)' != ''" />
64+
6665
<PackageVersion Update="NuGet.Configuration" Version="$(NuGetConfigurationVersion)" Condition="'$(NuGetConfigurationVersion)' != ''" />
6766
<PackageVersion Update="NuGet.Credentials" Version="$(NuGetCredentialsVersion)" Condition="'$(NuGetCredentialsVersion)' != ''" />
6867
<PackageVersion Update="NuGet.Protocol" Version="$(NuGetProtocolVersion)" Condition="'$(NuGetProtocolVersion)' != ''" />

‎src/Microsoft.TemplateEngine.Edge/BuiltInManagedProvider/GlobalSettings.cs‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,13 @@
11
// Licensed to the .NET Foundation under one or more agreements.
22
// The .NET Foundation licenses this file to you under the MIT license.
33

4+
using System.Text.Json;
5+
using System.Text.Json.Nodes;
46
using Microsoft.Extensions.Logging;
7+
58
using Microsoft.TemplateEngine.Abstractions;
69
using Microsoft.TemplateEngine.Abstractions.Installer;
710
using Microsoft.TemplateEngine.Edge.Settings;
8-
using Newtonsoft.Json;
9-
using Newtonsoft.Json.Linq;
1011

1112
namespace Microsoft.TemplateEngine.Edge.BuiltInManagedProvider
1213
{
@@ -82,20 +83,20 @@ public async Task<IReadOnlyList<TemplatePackageData>> GetInstalledTemplatePackag
8283
var jObject = _environmentSettings.Host.FileSystem.ReadObject(_globalSettingsFile);
8384
var packages = new List<TemplatePackageData>();
8485

85-
foreach (var package in jObject.Get<JArray>(nameof(GlobalSettingsData.Packages)) ?? new JArray())
86+
foreach (var package in jObject.Get<JsonArray>(nameof(GlobalSettingsData.Packages)) ?? new JsonArray())
8687
{
8788
packages.Add(new TemplatePackageData(
88-
package.ToGuid(nameof(TemplatePackageData.InstallerId)),
89-
package.Value<string>(nameof(TemplatePackageData.MountPointUri)) ?? string.Empty,
90-
((DateTime?)package[nameof(TemplatePackageData.LastChangeTime)]) ?? default,
89+
package!.ToGuid(nameof(TemplatePackageData.InstallerId)),
90+
package.ToString(nameof(TemplatePackageData.MountPointUri)) ?? string.Empty,
91+
package![nameof(TemplatePackageData.LastChangeTime)]?.GetValue<DateTime>() ?? default,
9192
package.ToStringDictionary(propertyName: nameof(TemplatePackageData.Details))));
9293
}
9394

9495
return packages;
9596
}
96-
catch (JsonReaderException ex)
97+
catch (JsonException ex)
9798
{
98-
var wrappedEx = new JsonReaderException(string.Format(LocalizableStrings.GlobalSettings_Error_CorruptedSettings, _globalSettingsFile, ex.Message), ex);
99+
var wrappedEx = new JsonException(string.Format(LocalizableStrings.GlobalSettings_Error_CorruptedSettings, _globalSettingsFile, ex.Message), ex.Path, ex.LineNumber, ex.BytePositionInLine, ex);
99100
throw wrappedEx;
100101
}
101102
catch (Exception)

‎src/Microsoft.TemplateEngine.Edge/BuiltInManagedProvider/GlobalSettingsData.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
// Licensed to the .NET Foundation under one or more agreements.
22
// The .NET Foundation licenses this file to you under the MIT license.
33

4+
using System.Text.Json.Serialization;
45
using Microsoft.TemplateEngine.Abstractions.Installer;
5-
using Newtonsoft.Json;
66

77
namespace Microsoft.TemplateEngine.Edge.BuiltInManagedProvider
88
{
@@ -16,7 +16,7 @@ internal GlobalSettingsData(IReadOnlyList<TemplatePackageData> packages)
1616
Packages = packages;
1717
}
1818

19-
[JsonProperty]
19+
[JsonInclude]
2020
internal IReadOnlyList<TemplatePackageData> Packages { get; }
2121
}
2222
}

‎src/Microsoft.TemplateEngine.Edge/Constraints/ConstraintsExtensions.cs‎

Lines changed: 25 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
// Licensed to the .NET Foundation under one or more agreements.
22
// The .NET Foundation licenses this file to you under the MIT license.
33

4+
using System.Text.Json;
5+
using System.Text.Json.Nodes;
46
using Microsoft.TemplateEngine.Utils;
5-
using Newtonsoft.Json.Linq;
67

78
namespace Microsoft.TemplateEngine.Edge.Constraints
89
{
@@ -16,23 +17,29 @@ internal static class Extensions
1617
/// <exception cref="ConfigurationException">Thrown on unexpected input - not a valid json string or array of string or an empty array.</exception>
1718
public static IEnumerable<string> ParseArrayOfConstraintStrings(this string? args)
1819
{
19-
JToken token = ParseConstraintJToken(args);
20+
JsonNode token = ParseConstraintJsonNode(args);
2021

21-
if (token.Type == JTokenType.String)
22+
if (token.GetValueKind() == JsonValueKind.String)
2223
{
23-
return new[] { token.Value<string>() ?? throw new ConfigurationException(string.Format(LocalizableStrings.Constraint_Error_ArgumentHasEmptyString, args)) };
24+
return new[] { token.GetValue<string>() ?? throw new ConfigurationException(string.Format(LocalizableStrings.Constraint_Error_ArgumentHasEmptyString, args)) };
2425
}
2526

26-
JArray array = token.ToConstraintsJArray(args, true);
27+
JsonArray array = token.ToConstraintsJsonArray(args, true);
2728

28-
return array.Values<string>().Select(value =>
29+
return array.Select(value =>
2930
{
30-
if (string.IsNullOrEmpty(value))
31+
if (value == null || value.GetValueKind() != JsonValueKind.String)
32+
{
33+
throw new ConfigurationException(string.Format(LocalizableStrings.Constraint_Error_ArgumentHasEmptyString, args));
34+
}
35+
36+
string? strValue = value.GetValue<string>();
37+
if (string.IsNullOrEmpty(strValue))
3138
{
3239
throw new ConfigurationException(string.Format(LocalizableStrings.Constraint_Error_ArgumentHasEmptyString, args));
3340
}
3441

35-
return value!;
42+
return strValue!;
3643
});
3744
}
3845

@@ -42,14 +49,14 @@ public static IEnumerable<string> ParseArrayOfConstraintStrings(this string? arg
4249
/// <param name="args">Input configuration string.</param>
4350
/// <returns>Enumeration of parsed JObject tokens.</returns>
4451
/// <exception cref="ConfigurationException">Thrown on unexpected input - not a valid json array or an empty array.</exception>
45-
public static IEnumerable<JObject> ParseArrayOfConstraintJObjects(this string? args)
52+
public static IEnumerable<JsonObject> ParseArrayOfConstraintJObjects(this string? args)
4653
{
47-
JToken token = ParseConstraintJToken(args);
48-
JArray array = token.ToConstraintsJArray(args, false);
54+
JsonNode token = ParseConstraintJsonNode(args);
55+
JsonArray array = token.ToConstraintsJsonArray(args, false);
4956

5057
return array.Select(value =>
5158
{
52-
if (value is not JObject jObj)
59+
if (value is not JsonObject jObj)
5360
{
5461
throw new ConfigurationException(string.Format(LocalizableStrings.Constraint_Error_InvalidJsonArray_Objects, args));
5562
}
@@ -103,29 +110,29 @@ public static IVersionSpecification ParseVersionSpecification(this string versio
103110
return versionInstance;
104111
}
105112

106-
private static JToken ParseConstraintJToken(this string? args)
113+
private static JsonNode ParseConstraintJsonNode(this string? args)
107114
{
108115
if (string.IsNullOrWhiteSpace(args))
109116
{
110117
throw new ConfigurationException(LocalizableStrings.Constraint_Error_ArgumentsNotSpecified);
111118
}
112119

113-
JToken? token;
120+
JsonNode? token;
114121
try
115122
{
116-
token = JToken.Parse(args!);
123+
token = JsonNode.Parse(args!);
117124
}
118125
catch (Exception e)
119126
{
120127
throw new ConfigurationException(string.Format(LocalizableStrings.Constraint_Error_InvalidJson, args), e);
121128
}
122129

123-
return token;
130+
return token ?? throw new ConfigurationException(string.Format(LocalizableStrings.Constraint_Error_InvalidJson, args));
124131
}
125132

126-
private static JArray ToConstraintsJArray(this JToken token, string? args, bool isStringTypeAllowed)
133+
private static JsonArray ToConstraintsJsonArray(this JsonNode token, string? args, bool isStringTypeAllowed)
127134
{
128-
if (token is not JArray array)
135+
if (token is not JsonArray array)
129136
{
130137
throw new ConfigurationException(string.Format(
131138
isStringTypeAllowed

‎src/Microsoft.TemplateEngine.Edge/Constraints/HostConstraint.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
// Licensed to the .NET Foundation under one or more agreements.
22
// The .NET Foundation licenses this file to you under the MIT license.
33

4+
using System.Text.Json.Nodes;
45
using Microsoft.TemplateEngine.Abstractions;
56
using Microsoft.TemplateEngine.Abstractions.Constraints;
67
using Microsoft.TemplateEngine.Utils;
7-
using Newtonsoft.Json.Linq;
88

99
namespace Microsoft.TemplateEngine.Edge.Constraints
1010
{
@@ -71,7 +71,7 @@ private static IEnumerable<HostInformation> ParseArgs(string? args)
7171
{
7272
List<HostInformation> hostInformation = new List<HostInformation>();
7373

74-
foreach (JObject jObj in args.ParseArrayOfConstraintJObjects())
74+
foreach (JsonObject jObj in args.ParseArrayOfConstraintJObjects())
7575
{
7676
string? hostName = jObj.ToString("hostname");
7777
string? version = jObj.ToString("version");

‎src/Microsoft.TemplateEngine.Edge/Microsoft.TemplateEngine.Edge.csproj‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
<PackageReference Include="NuGet.Configuration" />
2929
<PackageReference Include="NuGet.Credentials" />
3030
<PackageReference Include="NuGet.Protocol" />
31-
<PackageReference Include="Newtonsoft.Json" />
31+
<PackageReference Include="System.Text.Json" Condition="'$(TargetFrameworkIdentifier)' != '.NETCoreApp'" />
3232
</ItemGroup>
3333

3434
<ItemGroup Condition="'$(TargetFrameworkIdentifier)' == '.NETFramework'">

‎src/Microsoft.TemplateEngine.Edge/Settings/SettingsStore.cs‎

Lines changed: 26 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,66 +1,67 @@
11
// Licensed to the .NET Foundation under one or more agreements.
22
// The .NET Foundation licenses this file to you under the MIT license.
33

4+
using System.Text.Json;
5+
using System.Text.Json.Nodes;
6+
using System.Text.Json.Serialization;
47
using Microsoft.TemplateEngine.Abstractions;
58
using Microsoft.TemplateEngine.Utils;
6-
using Newtonsoft.Json;
7-
using Newtonsoft.Json.Linq;
89

910
namespace Microsoft.TemplateEngine.Edge.Settings
1011
{
1112
internal class SettingsStore
1213
{
13-
internal SettingsStore(JObject? obj)
14+
internal SettingsStore(JsonObject? obj)
1415
{
1516
if (obj == null)
1617
{
1718
return;
1819
}
1920

20-
if (obj.TryGetValue(nameof(ComponentGuidToAssemblyQualifiedName), StringComparison.OrdinalIgnoreCase, out JToken? componentGuidToAssemblyQualifiedNameToken))
21+
if (obj.TryGetValueCaseInsensitive(nameof(ComponentGuidToAssemblyQualifiedName), out JsonNode? componentGuidToAssemblyQualifiedNameToken))
2122
{
22-
if (componentGuidToAssemblyQualifiedNameToken is JObject componentGuidToAssemblyQualifiedNameObject)
23+
if (componentGuidToAssemblyQualifiedNameToken is JsonObject componentGuidToAssemblyQualifiedNameObject)
2324
{
24-
foreach (JProperty entry in componentGuidToAssemblyQualifiedNameObject.Properties())
25+
foreach (var entry in componentGuidToAssemblyQualifiedNameObject)
2526
{
26-
if (entry.Value is { Type: JTokenType.String })
27+
if (entry.Value?.GetValueKind() == JsonValueKind.String)
2728
{
28-
ComponentGuidToAssemblyQualifiedName[entry.Name] = entry.Value.ToString();
29+
ComponentGuidToAssemblyQualifiedName[entry.Key] = entry.Value.GetValue<string>();
2930
}
3031
}
3132
}
3233
}
3334

34-
if (obj.TryGetValue(nameof(ProbingPaths), StringComparison.OrdinalIgnoreCase, out JToken? probingPathsToken))
35+
if (obj.TryGetValueCaseInsensitive(nameof(ProbingPaths), out JsonNode? probingPathsToken))
3536
{
36-
if (probingPathsToken is JArray probingPathsArray)
37+
if (probingPathsToken is JsonArray probingPathsArray)
3738
{
38-
foreach (JToken path in probingPathsArray)
39+
foreach (JsonNode? path in probingPathsArray)
3940
{
40-
if (path is { Type: JTokenType.String })
41+
if (path?.GetValueKind() == JsonValueKind.String)
4142
{
42-
ProbingPaths.Add(path.ToString());
43+
ProbingPaths.Add(path.GetValue<string>());
4344
}
4445
}
4546
}
4647
}
4748

48-
if (obj.TryGetValue(nameof(ComponentTypeToGuidList), StringComparison.OrdinalIgnoreCase, out JToken? componentTypeToGuidListToken))
49+
if (obj.TryGetValueCaseInsensitive(nameof(ComponentTypeToGuidList), out JsonNode? componentTypeToGuidListToken))
4950
{
50-
if (componentTypeToGuidListToken is JObject componentTypeToGuidListObject)
51+
if (componentTypeToGuidListToken is JsonObject componentTypeToGuidListObject)
5152
{
52-
foreach (JProperty entry in componentTypeToGuidListObject.Properties())
53+
foreach (var entry in componentTypeToGuidListObject)
5354
{
54-
if (entry.Value is JArray values)
55+
if (entry.Value is JsonArray values)
5556
{
5657
HashSet<Guid> set = new HashSet<Guid>();
57-
ComponentTypeToGuidList[entry.Name] = set;
58+
ComponentTypeToGuidList[entry.Key] = set;
5859

59-
foreach (JToken value in values)
60+
foreach (JsonNode? value in values)
6061
{
61-
if (value is { Type: JTokenType.String })
62+
if (value?.GetValueKind() == JsonValueKind.String)
6263
{
63-
if (Guid.TryParse(value.ToString(), out Guid id))
64+
if (Guid.TryParse(value.GetValue<string>(), out Guid id))
6465
{
6566
set.Add(id);
6667
}
@@ -72,13 +73,13 @@ internal SettingsStore(JObject? obj)
7273
}
7374
}
7475

75-
[JsonProperty]
76+
[JsonInclude]
7677
internal Dictionary<string, string> ComponentGuidToAssemblyQualifiedName { get; } = new();
7778

78-
[JsonProperty]
79+
[JsonInclude]
7980
internal HashSet<string> ProbingPaths { get; } = new();
8081

81-
[JsonProperty]
82+
[JsonInclude]
8283
internal Dictionary<string, HashSet<Guid>> ComponentTypeToGuidList { get; } = new();
8384

8485
internal static SettingsStore Load(IEngineEnvironmentSettings engineEnvironmentSettings, SettingsFilePaths paths)
@@ -88,7 +89,7 @@ internal static SettingsStore Load(IEngineEnvironmentSettings engineEnvironmentS
8889
return new SettingsStore(null);
8990
}
9091

91-
JObject parsed;
92+
JsonObject parsed;
9293
using (Timing.Over(engineEnvironmentSettings.Host.Logger, "Parse settings"))
9394
{
9495
try

0 commit comments

Comments
 (0)