OCC-310: Fetch the log contents but only include the entries that will throw

Lombiq.Tests.UI/CompatibilitySuppressions.xml

@@ -316,6 +316,13 @@
     <Right>lib/net8.0/Lombiq.Tests.UI.dll</Right>
     <IsBaselineSuppression>true</IsBaselineSuppression>
   </Suppression>
+  <Suppression>
+    <DiagnosticId>CP0002</DiagnosticId>
+    <Target>M:Lombiq.Tests.UI.Services.SmtpServiceRunningContext.#ctor(System.Int32,System.Uri)</Target>
+    <Left>lib/net8.0/Lombiq.Tests.UI.dll</Left>
+    <Right>lib/net8.0/Lombiq.Tests.UI.dll</Right>
+    <IsBaselineSuppression>true</IsBaselineSuppression>
+  </Suppression>
   <Suppression>
     <DiagnosticId>CP0002</DiagnosticId>
     <Target>M:Lombiq.Tests.UI.Services.SynchronizingWebApplicationSnapshotManager.RunOperationAndSnapshotIfNewAsync(Lombiq.Tests.UI.Services.AppInitializer)</Target>
@@ -421,11 +428,4 @@
     <Right>lib/net8.0/Lombiq.Tests.UI.dll</Right>
     <IsBaselineSuppression>true</IsBaselineSuppression>
   </Suppression>
-  <Suppression>
-    <DiagnosticId>CP0002</DiagnosticId>
-    <Target>M:Lombiq.Tests.UI.Services.SmtpServiceRunningContext.#ctor(System.Int32,System.Uri)</Target>
-    <Left>lib/net8.0/Lombiq.Tests.UI.dll</Left>
-    <Right>lib/net8.0/Lombiq.Tests.UI.dll</Right>
-    <IsBaselineSuppression>true</IsBaselineSuppression>
-  </Suppression>
-</Suppressions>
+</Suppressions>

Lombiq.Tests.UI/Extensions/ApplicationLogEnumerableExtensions.cs

@@ -1,3 +1,4 @@
+using Lombiq.Tests.UI.Models;
 using Lombiq.Tests.UI.Services;
 using System;
 using System.Collections.Generic;
@@ -8,23 +9,27 @@ namespace Lombiq.Tests.UI.Extensions;
 
 public static class ApplicationLogEnumerableExtensions
 {
-    public static async Task<string> ToFormattedStringAsync(this IEnumerable<IApplicationLog> logs)
+    public static string ToFormattedStringCached(this IEnumerable<MemoryApplicationLog> logs)
     {
         var logsArray = logs.ToArray();
 
         if (logsArray.Length == 1)
         {
-            return Environment.NewLine + await LogLinesToFormattedStringAsync(logsArray[0]);
+            return Environment.NewLine + logsArray[0].ToFormattedString();
         }
 
-        // Parallelization with Task.WhenAll() isn't really necessary for performance here but would potentially change
-        // the order of the logs in the output.
-        var logContents = logsArray.AwaitEachAsync(async log =>
-            $"# Log name: {log.Name}" + Environment.NewLine + Environment.NewLine + await LogLinesToFormattedStringAsync(log));
+        var logContents = logsArray.Select(log =>
+            $"# Log name: {log.Name}" + Environment.NewLine + Environment.NewLine + log.ToFormattedString());
 
         return string.Join(Environment.NewLine + Environment.NewLine, logContents);
     }
 
-    private static async Task<string> LogLinesToFormattedStringAsync(IApplicationLog log) =>
-        string.Join(Environment.NewLine, (await log.GetEntriesAsync()).Select(logEntry => logEntry.ToString()));
+    public static async Task<string> ToFormattedStringAsync(this IEnumerable<IApplicationLog> logs)
+    {
+        var cached = await logs.AwaitEachAsync(log => MemoryApplicationLog.FromLogAsync(log));
+        return cached.ToFormattedStringCached();
+    }
+
+    private static string ToFormattedString(this MemoryApplicationLog log) =>
+        string.Join(Environment.NewLine, log.Entries.Select(logEntry => logEntry.ToString()));
 }

Lombiq.Tests.UI/Extensions/WebApplicationInstanceExtensions.cs

@@ -1,3 +1,4 @@
+using Lombiq.Tests.UI.Models;
 using Lombiq.Tests.UI.Services;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
@@ -30,7 +31,7 @@ public static async Task LogsShouldBeEmptyAsync(
         this IWebApplicationInstance webApplicationInstance,
         CancellationToken cancellationToken = default)
     {
-        var logs = await webApplicationInstance.GetLogsAsync(cancellationToken);
+        var logs = (await webApplicationInstance.GetLogsAsync(cancellationToken)).AsList();
         logs.ShouldNotContain(log => log.EntryCount > 0, await logs.ToFormattedStringAsync());
     }
 
@@ -166,8 +167,12 @@ private static async Task AssertLogsAsync(
         Action<IEnumerable<IApplicationLogEntry>, Expression<Func<IApplicationLogEntry, bool>>, string> shouldlyMethod,
         CancellationToken cancellationToken = default)
     {
-        var logs = await webApplicationInstance.GetLogsAsync(cancellationToken);
-        var logContents = await logs.ToFormattedStringAsync();
+        // Fetch the log contents but only include the entries that will throw below so you don't have to manually sort
+        // through ignored log entries if there is an error.
+        var logs = await (await webApplicationInstance.GetLogsAsync(cancellationToken))
+            .AwaitEachAsync(log => MemoryApplicationLog.FromLogAsync(log, logEntryPredicate.Compile()));
+
+        var logContents = logs.ToFormattedStringCached();
 
         foreach (var log in logs)
         {

Lombiq.Tests.UI/Models/MemoryApplicationLog.cs

@@ -0,0 +1,51 @@
+using Lombiq.Tests.UI.Services;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+
+namespace Lombiq.Tests.UI.Models;
+
+/// <summary>
+/// An <see cref="IApplicationLog"/> implementation where the entries are stored in memory. This makes the logs easier to inspect in IDE during
+/// debugging and working with a list of this type is easier to filter and aggregate than the base type.
+/// </summary>
+public record MemoryApplicationLog(string Name, IList<IApplicationLogEntry> Entries)
+    : IApplicationLog
+{
+    public int EntryCount => Entries.Count;
+
+    public Task<IEnumerable<IApplicationLogEntry>> GetEntriesAsync() => Task.FromResult(Entries.AsEnumerable());
+    public Task RemoveAsync() => throw new NotSupportedException();
+
+    /// <summary>
+    /// Converts an existing <paramref name="log"/> into in-memory log by eagerly fetching its contents. Use this to
+    /// avoid having to <see langword="await"/> <see cref="IApplicationLog.GetEntriesAsync"/> multiple times.
+    /// </summary>
+    public static Task<MemoryApplicationLog> FromLogAsync(
+        IApplicationLog log,
+        Func<IApplicationLogEntry, bool> predicate = null)
+    {
+        if (log.EntryCount < 1)
+        {
+            return Task.FromResult(new MemoryApplicationLog(log.Name, []));
+        }
+
+        if (log is MemoryApplicationLog cached)
+        {
+            return Task.FromResult(new MemoryApplicationLog(log.Name, FilterEntries(cached.Entries, predicate)));
+        }
+
+        return FromLogInnerAsync(log, predicate);
+    }
+
+    private static async Task<MemoryApplicationLog> FromLogInnerAsync(
+        IApplicationLog log,
+        Func<IApplicationLogEntry, bool> predicate) =>
+        new(log.Name, FilterEntries(await log.GetEntriesAsync(), predicate));
+
+    private static IList<IApplicationLogEntry> FilterEntries(
+        IEnumerable<IApplicationLogEntry> entries,
+        Func<IApplicationLogEntry, bool> predicate) =>
+        predicate is null ? entries.AsList() : entries.Where(predicate).ToList();
+}

Lombiq.Tests.UI/SecurityScanning/OrchardCoreUITestExecutorConfigurationExtensions.cs

@@ -34,14 +34,15 @@ public static OrchardCoreUITestExecutorConfiguration UseAssertAppLogsForSecurity
     /// </summary>
     public static Func<IWebApplicationInstance, Task> CreateAppLogAssertionForSecurityScan(params string[] additionalPermittedErrorLinePatterns)
     {
+        const string unhandledExceptionRegex = @"An unhandled exception has occurred while executing the request.\s*";
         var permittedErrorLinePatterns = new List<string>
         {
             // The model binding will throw FormatException exception with this text during ZAP active scan, when the
             // bot tries to send malicious query strings or POST data that doesn't fit the types expected by the model.
             // This is correct, safe behavior and should be logged in production.
             "is not a valid value for Boolean",
-            "An unhandled exception has occurred while executing the request. System.FormatException: any",
-            "System.FormatException: The input string '[\\S\\s]+' was not in a correct format.",
+            unhandledExceptionRegex + "System.FormatException: any",
+            @"System.FormatException: The input string '[\S\s]+' was not in a correct format.",
             "System.FormatException: The input string 'any",
             // Happens when the static file middleware tries to access a path that doesn't exist or access a file as a
             // directory. Presumably this is an attempt to access protected files using source path manipulation. This
@@ -52,7 +53,8 @@ public static Func<IWebApplicationInstance, Task> CreateAppLogAssertionForSecuri
             // This happens when a request's model contains a dictionary and a key is missing. While this can be a
             // legitimate application error, during a security scan it's more likely the result of an incomplete
             // artificially constructed request. So the means the ASP.NET Core model binding is working as intended.
-            "An unhandled exception has occurred while executing the request. System.ArgumentNullException: Value cannot be null. (Parameter 'key')",
+            unhandledExceptionRegex + "System.ArgumentNullException: Value cannot be null. (Parameter 'key')",
+            "at Microsoft.AspNetCore.Mvc.ModelBinding",
             // One way to verify correct error handling is to navigate to ~/Lombiq.Tests.UI.Shortcuts/Error/Index, which
             // always throws an exception. This also gets logged but it's expected, so it should be ignored.
             ErrorController.ExceptionMessage,
@@ -69,10 +71,10 @@ public static Func<IWebApplicationInstance, Task> CreateAppLogAssertionForSecuri
         return app =>
             app.LogsShouldNotContainAsync(
                 logEntry =>
-                    !permittedErrorLinePatterns.Any(pattern =>
-                        Regex.IsMatch(logEntry.ToString(), pattern, RegexOptions.IgnoreCase | RegexOptions.Compiled)) &&
+                    logEntry.Level >= LogLevel.Error &&
                     AppLogAssertionHelper.NotMediaCacheEntries(logEntry) &&
-                    logEntry.Level >= LogLevel.Error,
+                    !permittedErrorLinePatterns.Any(pattern =>
+                        Regex.IsMatch(logEntry.ToString(), pattern, RegexOptions.IgnoreCase | RegexOptions.Compiled)),
                 TestContext.Current.CancellationToken);
     }
 }