diff --git a/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs index 5b7e0cda..d08cb1de 100644 --- a/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs +++ b/src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs @@ -15,6 +15,7 @@ using System; using System.Buffers; using System.Diagnostics.CodeAnalysis; +using System.IO; using System.Linq; using System.Net; using System.Text; @@ -45,7 +46,7 @@ public IngestionEndpoints(ForwardingAuthenticationStrategy forwardingChannels, S _forwardingChannels = forwardingChannels; _config = config; } - + public void MapEndpoints(WebApplication app) { app.MapPost("ingest/clef", (Delegate) (async (HttpContext context) => await IngestCompactFormatAsync(context))); @@ -63,15 +64,17 @@ async Task IngestAsync(HttpContext context) if (contentType != null && contentType.StartsWith(clefMediaType)) return await IngestCompactFormatAsync(context); - IngestionLog.ForClient(context.Connection.RemoteIpAddress) - .Error("Client supplied a legacy raw-format (non-CLEF) payload"); - return Error(HttpStatusCode.BadRequest, "Only newline-delimited JSON (CLEF) payloads are supported."); + return await IngestLegacyRawFormatAsync(context); + + // IngestionLog.ForClient(context.Connection.RemoteIpAddress) + // .Error("Client supplied a legacy raw-format (non-CLEF) payload"); + // return Error(HttpStatusCode.BadRequest, "Only newline-delimited JSON (CLEF) payloads are supported."); } - + async Task IngestCompactFormatAsync(HttpContext context) { byte[]? rented = null; - + try { using var cts = CancellationTokenSource.CreateLinkedTokenSource(context.RequestAborted); @@ -177,6 +180,190 @@ async Task IngestCompactFormatAsync(HttpContext context) } } + async Task IngestLegacyRawFormatAsync(HttpContext context) + { + try + { + using var cts = CancellationTokenSource.CreateLinkedTokenSource(context.RequestAborted); + cts.CancelAfter(TimeSpan.FromSeconds(5)); + + var requestApiKey = GetApiKey(context.Request); + var log = _forwardingChannels.GetForwardingChannel(requestApiKey); + + // Read the entire request body + using var reader = new StreamReader(context.Request.Body); + var requestBody = await reader.ReadToEndAsync(cts.Token); + + // Validate and parse the legacy raw format + JsonDocument jsonDoc; + try + { + jsonDoc = JsonDocument.Parse(requestBody); + } + catch (JsonException) + { + IngestionLog.ForPayload(context.Connection.RemoteIpAddress, requestBody) + .Error("Payload validation failed: JSON parsing failure"); + return Error(HttpStatusCode.BadRequest, "Payload validation failed: JSON parsing failure."); + } + + // Extract events array + if (!jsonDoc.RootElement.TryGetProperty("Events", out var eventsArray) && + !jsonDoc.RootElement.TryGetProperty("events", out eventsArray)) + { + IngestionLog.ForPayload(context.Connection.RemoteIpAddress, requestBody) + .Error("Payload validation failed: events were not found in raw payload"); + return Error(HttpStatusCode.BadRequest, "Payload validation failed: events were not found in raw payload."); + } + + if (eventsArray.ValueKind != JsonValueKind.Array) + { + IngestionLog.ForPayload(context.Connection.RemoteIpAddress, requestBody) + .Error("Payload validation failed: events must be an array"); + return Error(HttpStatusCode.BadRequest, "Payload validation failed: events must be an array."); + } + + // Convert each legacy event to CLEF + using var clefStream = new MemoryStream(); + foreach (var evt in eventsArray.EnumerateArray()) + { + ConvertLegacyEventToClef(evt, clefStream); + } + + if (clefStream.Length > 0) + { + await log.WriteAsync(clefStream.ToArray(), cts.Token); + } + + return SuccessfulIngestion(); + } + catch (Exception ex) + { + IngestionLog.ForClient(context.Connection.RemoteIpAddress) + .Error(ex, "Ingestion failed"); + return Error(HttpStatusCode.InternalServerError, "Ingestion failed."); + } + } + + static void ConvertLegacyEventToClef(JsonElement legacyEvent, Stream output) + { + using var writer = new Utf8JsonWriter(output, new JsonWriterOptions { SkipValidation = true }); + writer.WriteStartObject(); + + string? timestamp = null; + string? level = null; + string? messageTemplate = null; + string? exception = null; + JsonElement? properties = null; + + // Extract known fields from legacy event + foreach (var prop in legacyEvent.EnumerateObject()) + { + switch (prop.Name) + { + case "Timestamp": + timestamp = prop.Value.GetString(); + break; + case "Level": + level = prop.Value.GetString(); + break; + case "MessageTemplate": + messageTemplate = prop.Value.GetString(); + break; + case "Exception": + if (prop.Value.ValueKind != JsonValueKind.Null) + exception = prop.Value.GetString(); + break; + case "Properties": + properties = prop.Value; + break; + } + } + + // Write CLEF fields + if (!string.IsNullOrEmpty(timestamp)) + { + writer.WriteString("@t", timestamp); + } + + if (!string.IsNullOrEmpty(level)) + { + writer.WriteString("@l", level); + } + + if (!string.IsNullOrEmpty(messageTemplate)) + { + writer.WriteString("@mt", messageTemplate); + } + + if (!string.IsNullOrEmpty(exception)) + { + writer.WriteString("@x", exception); + } + + // Write properties as top-level fields + if (properties.HasValue) + { + foreach (var prop in properties.Value.EnumerateObject()) + { + writer.WritePropertyName(prop.Name); + WriteJsonElement(writer, prop.Value); + } + } + + writer.WriteEndObject(); + writer.Flush(); + + // Write newline after each event + output.WriteByte((byte)'\n'); + } + + static void WriteJsonElement(Utf8JsonWriter writer, JsonElement element) + { + switch (element.ValueKind) + { + case JsonValueKind.Object: + writer.WriteStartObject(); + foreach (var prop in element.EnumerateObject()) + { + writer.WritePropertyName(prop.Name); + WriteJsonElement(writer, prop.Value); + } + writer.WriteEndObject(); + break; + case JsonValueKind.Array: + writer.WriteStartArray(); + foreach (var item in element.EnumerateArray()) + { + WriteJsonElement(writer, item); + } + writer.WriteEndArray(); + break; + case JsonValueKind.String: + writer.WriteStringValue(element.GetString()); + break; + case JsonValueKind.Number: + if (element.TryGetInt32(out var intValue)) + writer.WriteNumberValue(intValue); + else if (element.TryGetInt64(out var longValue)) + writer.WriteNumberValue(longValue); + else if (element.TryGetDouble(out var doubleValue)) + writer.WriteNumberValue(doubleValue); + else + writer.WriteRawValue(element.GetRawText()); + break; + case JsonValueKind.True: + writer.WriteBooleanValue(true); + break; + case JsonValueKind.False: + writer.WriteBooleanValue(false); + break; + case JsonValueKind.Null: + writer.WriteNullValue(); + break; + } + } + static bool DefaultedBoolQuery(HttpRequest request, string queryParameterName) { var parameter = request.Query[queryParameterName]; @@ -207,13 +394,13 @@ bool ValidateClef(Span evt, [NotNullWhen(false)] out string? errorFragment { // Note that `errorFragment` does not include user-supplied values; we opt in to adding this to // the ingestion log and include it using `ForPayload()`. - + if (evt.Length > _config.Connection.EventSizeLimitBytes) { errorFragment = "an event exceeds the configured size limit"; return false; } - + var reader = new Utf8JsonReader(evt); var foundTimestamp = false; @@ -269,7 +456,39 @@ bool ValidateClef(Span evt, [NotNullWhen(false)] out string? errorFragment errorFragment = null; return true; } - + + bool ValidateRaw(Span evt, [NotNullWhen(false)] out string? errorFragment) + { + // Note that `errorFragment` does not include user-supplied values; we opt in to adding this to + // the ingestion log and include it using `ForPayload()`. + + if (evt.Length > _config.Connection.EventSizeLimitBytes) + { + errorFragment = "an event exceeds the configured size limit"; + return false; + } + + try + { + + var jsonDoc = JsonDocument.Parse(evt.ToArray()); + if (!(jsonDoc.RootElement.TryGetProperty("events", out var eventsToken) || + jsonDoc.RootElement.TryGetProperty("Events", out eventsToken))) + { + errorFragment = "events were not found in raw payload"; + return false; + } + } + catch (JsonException) + { + errorFragment = "JSON parsing failure"; + return false; + } + + errorFragment = null; + return true; + } + static IResult Error(HttpStatusCode statusCode, string message) { return Results.Json(new ErrorPart { Error = message }, statusCode: (int)statusCode); @@ -279,8 +498,8 @@ static IResult SuccessfulIngestion() { return TypedResults.Content( "{}", - "application/json", - Utf8, + "application/json", + Utf8, StatusCodes.Status201Created); } } diff --git a/test/SeqCli.EndToEnd/Forwarder/ForwarderLegacyRawIngestionTestCase.cs b/test/SeqCli.EndToEnd/Forwarder/ForwarderLegacyRawIngestionTestCase.cs new file mode 100644 index 00000000..ed432ad4 --- /dev/null +++ b/test/SeqCli.EndToEnd/Forwarder/ForwarderLegacyRawIngestionTestCase.cs @@ -0,0 +1,53 @@ +using System; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; +using Seq.Api; +using SeqCli.EndToEnd.Support; +using Serilog; +using Xunit; + +namespace SeqCli.EndToEnd.Forwarder; + +public class ForwarderLegacyRawIngestionTestCase : ICliTestCase +{ + public async Task ExecuteAsync(SeqConnection _, ILogger logger, CliCommandRunner runner) + { + var (forwarder, forwarderUri) = await runner.SpawnForwarderAsync(); + using (forwarder) + { + using var connection = new SeqConnection(forwarderUri); + + // Test case from problem statement + var legacyRawPayload = @"{""Events"":[{""Timestamp"":""2026-02-02T18:32:53.0512280+00:00"",""Level"":""Information"",""Exception"":null,""MessageTemplate"":""[{Step}] succeeded ({Duration} seconds)"",""Properties"":{""Application"":""SerilogPS PS2"",""Environment"":""Prod"",""Step"":""tudo pronto"",""Duration"":0.0,""Version"":""2.3"",""Host"":null}}]}"; + + await IngestLegacyRaw(connection, legacyRawPayload, HttpStatusCode.Created); + + // Test with multiple events + var multipleEvents = @"{""Events"":[ + {""Timestamp"":""2026-02-02T18:32:53.0512280+00:00"",""Level"":""Information"",""Exception"":null,""MessageTemplate"":""Event 1"",""Properties"":{""Prop1"":""Value1""}}, + {""Timestamp"":""2026-02-02T18:32:54.0512280+00:00"",""Level"":""Warning"",""Exception"":null,""MessageTemplate"":""Event 2"",""Properties"":{""Prop2"":42}} + ]}"; + + await IngestLegacyRaw(connection, multipleEvents, HttpStatusCode.Created); + + // Test with exception + var withException = @"{""Events"":[{""Timestamp"":""2026-02-02T18:32:53.0512280+00:00"",""Level"":""Error"",""Exception"":""System.Exception: Test exception"",""MessageTemplate"":""An error occurred"",""Properties"":{}}]}"; + + await IngestLegacyRaw(connection, withException, HttpStatusCode.Created); + + // Test with nested properties + var withNestedProps = @"{""Events"":[{""Timestamp"":""2026-02-02T18:32:53.0512280+00:00"",""Level"":""Debug"",""Exception"":null,""MessageTemplate"":""Complex event"",""Properties"":{""Nested"":{""Inner"":""Value""},""Array"":[1,2,3]}}]}"; + + await IngestLegacyRaw(connection, withNestedProps, HttpStatusCode.Created); + } + } + + static async Task IngestLegacyRaw(SeqConnection connection, string rawPayload, HttpStatusCode expectedStatusCode) + { + var content = new StringContent(rawPayload, Encoding.UTF8, "application/json"); + var response = await connection.Client.HttpClient.PostAsync("api/events/raw", content); + Assert.Equal(expectedStatusCode, response.StatusCode); + } +}