fix: address STJ migration bugs and PR review feedback

- Fix ConvertJsonElement ternary type coercion: (object)l cast prevents
  implicit long→double widening in switch expression
- Make ObjectToInferredTypesConverter configurable with preferInt64 flag
  for ES serializer to match JSON.NET DataObjectConverter behavior
- Fix ElasticSystemTextJsonSerializer: remove ReadStreamToSpan lifetime
  bug (span backed by disposed MemoryStream), deserialize from stream
  directly with MemoryStream fast-path
- Fix Serialize<T> indentation: pass JsonWriterOptions to Utf8JsonWriter
  so SerializationFormatting.Indented actually produces indented output
- Handle exponent notation (1e5) as floating-point in ReadNumber
- Use double consistently (not decimal) for floating-point to match
  JSON.NET behavior
- Fix RenameAll return value: return whether any renames occurred
- Add using var to MemoryStream in EventController and EventPostsJob
- Handle empty response bodies in SendRequestAsAsync (STJ throws on
  empty input, Newtonsoft returned default)
- Fix SerializerTests: put unknown properties at root level to test
  JsonExtensionData→ConvertJsonElement path correctly
- Revert AGENTS.md to main

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: niemyjski

AGENTS.md

@@ -53,26 +53,3 @@ Load from `.github/skills/<name>/SKILL.md` when working in that domain:
 - Use `npm ci` (not `npm install`)
 - Never commit secrets — use environment variables
 - NuGet feeds are in `NuGet.Config` — don't add sources
-
-## Serialization Architecture
-
-The project uses **System.Text.Json (STJ)** exclusively. NEST still brings in Newtonsoft.Json transitively, but all application-level serialization uses STJ:
-
-| Component      | Serializer                        | Notes                                       |
-| -------------- | --------------------------------- | -------------------------------------------- |
-| Elasticsearch  | `ElasticSystemTextJsonSerializer` | Custom `IElasticsearchSerializer` using STJ  |
-| Event Upgrader | `System.Text.Json.Nodes`          | JsonObject/JsonArray for mutable DOM         |
-| Data Storage   | `SystemTextJsonSerializer`        | Via Foundatio's STJ support                  |
-| API            | STJ (built-in)                    | ASP.NET Core default with custom options     |
-
-**Key files:**
-
-- `ElasticSystemTextJsonSerializer.cs` - Custom `IElasticsearchSerializer` for NEST
-- `JsonNodeExtensions.cs` - STJ equivalents of JObject helpers
-- `ObjectToInferredTypesConverter.cs` - Handles JObject/JToken from NEST during STJ serialization
-- `V*_EventUpgrade.cs` - Event version upgraders using JsonObject
-
-**Security:**
-
-- Safe JSON encoding used everywhere (escapes `<`, `>`, `&`, `'` for XSS protection)
-- No `UnsafeRelaxedJsonEscaping` in the codebase

src/Exceptionless.Core/Extensions/DataDictionaryExtensions.cs

@@ -78,7 +78,7 @@ public static class DataDictionaryExtensions
                 if (result is not null)
                     return result;
             }
-            catch
+            catch (Exception ex) when (ex is JsonException or InvalidOperationException or FormatException)
             {
                 // Ignored - fall through to next handler
             }
@@ -94,7 +94,7 @@ public static class DataDictionaryExtensions
                 if (result is not null)
                     return result;
             }
-            catch
+            catch (Exception ex) when (ex is JsonException or InvalidOperationException or FormatException)
             {
                 // Ignored - fall through to next handler
             }

src/Exceptionless.Core/Extensions/JsonNodeExtensions.cs

@@ -1,7 +1,5 @@
-using System.Text.Encodings.Web;
 using System.Text.Json;
 using System.Text.Json.Nodes;
-using System.Text.Unicode;
 
 namespace Exceptionless.Core.Extensions;
 
@@ -11,28 +9,6 @@ namespace Exceptionless.Core.Extensions;
 /// </summary>
 public static class JsonNodeExtensions
 {
-    /// <summary>
-    /// XSS-safe encoder for JSON output formatting.
-    /// This encoder ensures proper XSS protection while allowing Unicode characters
-    /// for internationalization support.
-    ///
-    /// Security features:
-    /// - HTML-sensitive characters (&lt;, &gt;, &amp;) are escaped for XSS protection
-    /// - Single quotes are escaped as \u0027 (per ECMAScript spec)
-    /// - Control characters are escaped for security
-    /// </summary>
-    private static readonly JavaScriptEncoder SafeJsonEncoder = JavaScriptEncoder.Create(new TextEncoderSettings(UnicodeRanges.All));
-
-    /// <summary>
-    /// JSON options with safe XSS encoding for tests.
-    /// Validates that dangerous characters (&lt;, &gt;, &amp;, ') are properly escaped.
-    /// Production code should use <see cref="JsonSerializerOptions"/> from DI.
-    /// </summary>
-    internal static readonly JsonSerializerOptions SafeSerializerOptions = new()
-    {
-        Encoder = SafeJsonEncoder
-    };
-
     /// <summary>
     /// Checks if a JsonNode is null or empty (no values for objects/arrays).
     /// </summary>
@@ -222,7 +198,7 @@ public static bool RenameAll(this JsonObject target, string currentName, string
             obj.Rename(currentName, newName);
         }
 
-        return true;
+        return objectsWithProperty.Count > 0;
     }
 
     /// <summary>
@@ -293,14 +269,6 @@ public static bool RenameAll(this JsonObject target, string currentName, string
             yield return desc;
     }
 
-    /// <summary>
-    /// Converts an object to a JsonNode using System.Text.Json serialization.
-    /// </summary>
-    public static JsonNode? ToJsonNode<T>(T value, JsonSerializerOptions options)
-    {
-        return JsonSerializer.SerializeToNode(value, options);
-    }
-
     /// <summary>
     /// Checks if a JsonNode has any values (for objects: has properties, for arrays: has items).
     /// </summary>
@@ -331,14 +299,6 @@ public static bool HasValues(this JsonNode? node)
         return array.Deserialize<List<T>>(options);
     }
 
-    /// <summary>
-    /// Creates a JsonValue from a primitive value.
-    /// </summary>
-    public static JsonValue? CreateValue<T>(T value)
-    {
-        return JsonValue.Create(value);
-    }
-
     /// <summary>
     /// Converts a JsonNode to a pretty-printed JSON string.
     /// Uses 2-space indentation. Normalizes dates to match existing data format (Z → +00:00).

src/Exceptionless.Core/Jobs/EventPostsJob.cs

@@ -302,7 +302,7 @@ private async Task RetryEventsAsync(List<PersistentEvent> eventsToRetry, EventPo
         {
             try
             {
-                var stream = new MemoryStream(ev.GetBytes(_serializer));
+                using var stream = new MemoryStream(ev.GetBytes(_serializer));
 
                 // Put this single event back into the queue so we can retry it separately.
                 await _eventPostService.EnqueueAsync(new EventPost(false)

src/Exceptionless.Core/Models/Event.cs

@@ -98,7 +98,7 @@ void IJsonOnDeserialized.OnDeserialized()
         return element.ValueKind switch
         {
             JsonValueKind.String => element.GetString(),
-            JsonValueKind.Number => element.TryGetInt64(out long l) ? l : element.GetDouble(),
+            JsonValueKind.Number => element.TryGetInt64(out long l) ? (object)l : element.GetDouble(),
             JsonValueKind.True => true,
             JsonValueKind.False => false,
             JsonValueKind.Null or JsonValueKind.Undefined => null,

src/Exceptionless.Core/Plugins/EventUpgrader/Default/V2_EventUpgrade.cs

@@ -150,11 +150,8 @@ private void RenameAndValidateExtraExceptionProperties(string? id, JsonObject er
             var extraProperties = JsonNode.Parse(json) as JsonObject;
             if (extraProperties is not null)
             {
-                foreach (var property in extraProperties.ToList())
+                foreach (var property in extraProperties.ToList().Where(p => !p.Value.IsNullOrEmpty()))
                 {
-                    if (property.Value.IsNullOrEmpty())
-                        continue;
-
                     string dataKey = property.Key;
                     if (extendedData[dataKey] is not null)
                         dataKey = "_" + dataKey;
@@ -165,7 +162,7 @@ private void RenameAndValidateExtraExceptionProperties(string? id, JsonObject er
                 }
             }
         }
-        catch (Exception) { }
+        catch (JsonException) { }
 
         if (ext.IsNullOrEmpty())
             return;

src/Exceptionless.Core/Serialization/ElasticSystemTextJsonSerializer.cs

@@ -59,11 +59,19 @@ private static JsonSerializerOptions CreateOptions(JsonSerializerOptions? baseOp
         options.DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull;
         options.WriteIndented = writeIndented;
 
-        // Insert Elasticsearch converters at the beginning for priority
+        // Replace the default ObjectToInferredTypesConverter with one that returns Int64
+        // for all integers, matching JSON.NET DataObjectConverter behavior. This ensures
+        // Event.Data values round-trip through Elasticsearch with consistent types.
+        var defaultConverter = options.Converters.FirstOrDefault(c => c is ObjectToInferredTypesConverter);
+        if (defaultConverter is not null)
+            options.Converters.Remove(defaultConverter);
+        options.Converters.Insert(0, new ObjectToInferredTypesConverter(preferInt64: true));
+
+        // Insert Elasticsearch converters for priority
         // Order matters: more specific converters should come first
-        options.Converters.Insert(0, new DynamicDictionaryConverter());
-        options.Converters.Insert(1, new Iso8601DateTimeOffsetConverter());
-        options.Converters.Insert(2, new Iso8601DateTimeConverter());
+        options.Converters.Insert(1, new DynamicDictionaryConverter());
+        options.Converters.Insert(2, new Iso8601DateTimeOffsetConverter());
+        options.Converters.Insert(3, new Iso8601DateTimeConverter());
 
         return options;
     }
@@ -79,8 +87,11 @@ private JsonSerializerOptions GetOptions(SerializationFormatting formatting) =>
         if (IsEmptyStream(stream))
             return null;
 
-        var buffer = ReadStreamToSpan(stream);
-        return JsonSerializer.Deserialize(buffer, type, _optionsCompact.Value);
+        // Fast path: MemoryStream with accessible buffer avoids buffering
+        if (stream is MemoryStream ms && ms.TryGetBuffer(out var segment))
+            return JsonSerializer.Deserialize(segment.AsSpan((int)ms.Position), type, _optionsCompact.Value);
+
+        return JsonSerializer.Deserialize(stream, type, _optionsCompact.Value);
     }
 
     /// <inheritdoc />
@@ -89,15 +100,22 @@ private JsonSerializerOptions GetOptions(SerializationFormatting formatting) =>
         if (IsEmptyStream(stream))
             return default;
 
-        var buffer = ReadStreamToSpan(stream);
-        return JsonSerializer.Deserialize<T>(buffer, _optionsCompact.Value);
+        // Fast path: MemoryStream with accessible buffer avoids buffering
+        if (stream is MemoryStream ms && ms.TryGetBuffer(out var segment))
+            return JsonSerializer.Deserialize<T>(segment.AsSpan((int)ms.Position), _optionsCompact.Value);
+
+        return JsonSerializer.Deserialize<T>(stream, _optionsCompact.Value);
     }
 
     /// <inheritdoc />
     public void Serialize<T>(T data, Stream stream, SerializationFormatting formatting = SerializationFormatting.None)
     {
-        using var writer = new Utf8JsonWriter(stream);
         var options = GetOptions(formatting);
+        using var writer = new Utf8JsonWriter(stream, new JsonWriterOptions
+        {
+            Indented = formatting == SerializationFormatting.Indented,
+            Encoder = options.Encoder
+        });
 
         if (data is null)
         {
@@ -153,31 +171,10 @@ public Task SerializeAsync<T>(
 
     #endregion
 
-    #region Stream Helpers
-
     private static bool IsEmptyStream(Stream? stream)
     {
         return stream is null || stream == Stream.Null || (stream.CanSeek && stream.Length == 0);
     }
-
-    private static ReadOnlySpan<byte> ReadStreamToSpan(Stream stream)
-    {
-        // Fast path: if already a MemoryStream with accessible buffer, use it directly
-        if (stream is MemoryStream ms && ms.TryGetBuffer(out var segment))
-        {
-            return segment.AsSpan();
-        }
-
-        // Slow path: copy to new buffer
-        using var buffer = stream.CanSeek
-            ? new MemoryStream((int)stream.Length)
-            : new MemoryStream();
-
-        stream.CopyTo(buffer);
-        return buffer.TryGetBuffer(out var seg) ? seg.AsSpan() : buffer.ToArray();
-    }
-
-    #endregion
 }
 
 #region Elasticsearch-Specific Converters

src/Exceptionless.Core/Serialization/ObjectToInferredTypesConverter.cs

@@ -18,7 +18,7 @@ namespace Exceptionless.Core.Serialization;
 /// </para>
 /// <list type="bullet">
 ///   <item><description><c>true</c>/<c>false</c> → <see cref="bool"/></description></item>
-///   <item><description>Numbers → <see cref="long"/> (if fits) or <see cref="double"/></description></item>
+///   <item><description>Numbers → <see cref="int"/> (if fits), <see cref="long"/>, or <see cref="decimal"/>; with <c>preferInt64</c>, always <see cref="long"/> for integers and <see cref="double"/> for floats</description></item>
 ///   <item><description>Strings with ISO 8601 date format → <see cref="DateTimeOffset"/></description></item>
 ///   <item><description>Other strings → <see cref="string"/></description></item>
 ///   <item><description><c>null</c> → <c>null</c></description></item>
@@ -44,6 +44,25 @@ namespace Exceptionless.Core.Serialization;
 /// <seealso href="https://learn.microsoft.com/en-us/dotnet/standard/serialization/system-text-json/converters-how-to#deserialize-inferred-types-to-object-properties"/>
 public sealed class ObjectToInferredTypesConverter : JsonConverter<object?>
 {
+    private readonly bool _preferInt64;
+
+    /// <summary>
+    /// Initializes a new instance with default settings (integers that fit Int32 are returned as <see cref="int"/>).
+    /// </summary>
+    public ObjectToInferredTypesConverter() : this(preferInt64: false) { }
+
+    /// <summary>
+    /// Initializes a new instance with configurable integer handling.
+    /// </summary>
+    /// <param name="preferInt64">
+    /// When <c>true</c>, all integers are returned as <see cref="long"/> to match JSON.NET behavior.
+    /// Used by the Elasticsearch serializer to maintain compatibility with <c>DataObjectConverter</c>.
+    /// </param>
+    public ObjectToInferredTypesConverter(bool preferInt64)
+    {
+        _preferInt64 = preferInt64;
+    }
+
     /// <inheritdoc />
     public override object? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
     {
@@ -103,39 +122,45 @@ public override void Write(Utf8JsonWriter writer, object? value, JsonSerializerO
     ///   <item><description>Serializing back would lose the decimal representation</description></item>
     /// </list>
     /// </remarks>
-    private static object ReadNumber(ref Utf8JsonReader reader)
+    private object ReadNumber(ref Utf8JsonReader reader)
     {
         // Check the raw text to preserve decimal vs integer representation
         // This is critical for data integrity - 0.0 should stay as double, not become 0L
         ReadOnlySpan<byte> rawValue = reader.HasValueSequence
             ? reader.ValueSequence.ToArray()
             : reader.ValueSpan;
 
-        // If the raw text contains a decimal point, treat as floating-point
-        if (rawValue.Contains((byte)'.'))
+        // If the raw text contains a decimal point or exponent, treat as floating-point
+        if (rawValue.Contains((byte)'.') || rawValue.Contains((byte)'e') || rawValue.Contains((byte)'E'))
         {
-            // Try decimal for precise values (e.g., financial data) before double
-            if (reader.TryGetDecimal(out decimal d))
-                return d;
+            if (_preferInt64)
+                return reader.GetDouble();
 
-            // Fall back to double for floating-point
-            return reader.GetDouble();
+            return reader.GetDecimal();
         }
 
         // No decimal point - this is an integer
-        // Try int32 first for smaller values, then long for larger integers
-        if (reader.TryGetInt32(out int i))
-            return i;
+        if (_preferInt64)
+        {
+            // Match JSON.NET DataObjectConverter behavior: always return Int64
+            if (reader.TryGetInt64(out long l))
+                return l;
+        }
+        else
+        {
+            // Default STJ behavior: return smallest fitting integer type
+            if (reader.TryGetInt32(out int i))
+                return i;
 
-        if (reader.TryGetInt64(out long l))
-            return l;
+            if (reader.TryGetInt64(out long l))
+                return l;
+        }
 
-        // For very large integers, try decimal first to preserve precision
-        if (reader.TryGetDecimal(out decimal dec))
-            return dec;
+        // For very large integers that don't fit in long, fall back to decimal/double
+        if (_preferInt64)
+            return reader.GetDouble();
 
-        // Fall back to double only if decimal also fails
-        return reader.GetDouble();
+        return reader.GetDecimal();
     }
 
     /// <summary>
@@ -160,7 +185,7 @@ private static object ReadNumber(ref Utf8JsonReader reader)
     /// Uses <see cref="StringComparer.OrdinalIgnoreCase"/> for property name matching,
     /// consistent with <see cref="DataDictionary"/> behavior.
     /// </remarks>
-    private static Dictionary<string, object?> ReadObject(ref Utf8JsonReader reader, JsonSerializerOptions options)
+    private Dictionary<string, object?> ReadObject(ref Utf8JsonReader reader, JsonSerializerOptions options)
     {
         var dictionary = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase);
 
@@ -186,7 +211,7 @@ private static object ReadNumber(ref Utf8JsonReader reader)
     /// <summary>
     /// Recursively reads a JSON array into a <see cref="List{T}"/> of objects.
     /// </summary>
-    private static List<object?> ReadArray(ref Utf8JsonReader reader, JsonSerializerOptions options)
+    private List<object?> ReadArray(ref Utf8JsonReader reader, JsonSerializerOptions options)
     {
         var list = new List<object?>();
 
@@ -204,7 +229,7 @@ private static object ReadNumber(ref Utf8JsonReader reader)
     /// <summary>
     /// Reads a single JSON value of any type, dispatching to the appropriate reader method.
     /// </summary>
-    private static object? ReadValue(ref Utf8JsonReader reader, JsonSerializerOptions options)
+    private object? ReadValue(ref Utf8JsonReader reader, JsonSerializerOptions options)
     {
         return reader.TokenType switch
         {

src/Exceptionless.Web/Controllers/EventController.cs

@@ -1115,7 +1115,7 @@ private async Task<ActionResult> GetSubmitEventAsync(string? projectId = null, i
                 charSet = contentTypeHeader.Charset.ToString();
             }
 
-            var stream = new MemoryStream(ev.GetBytes(_serializer));
+            using var stream = new MemoryStream(ev.GetBytes(_serializer));
             await _eventPostService.EnqueueAsync(new EventPost(_appOptions.EnableArchive)
             {
                 ApiVersion = apiVersion,