diff --git a/.github/workflows/publish-cloudsmith.yml b/.github/workflows/publish-cloudsmith.yml index c92bc5923..e6203dced 100644 --- a/.github/workflows/publish-cloudsmith.yml +++ b/.github/workflows/publish-cloudsmith.yml @@ -8,7 +8,7 @@ on: jobs: publish-nuget: name: Publish to Cloudsmith - uses: Lombiq/GitHub-Actions/.github/workflows/publish-nuget.yml@dev + uses: Lombiq/GitHub-Actions/.github/workflows/publish-nuget.yml@issue/OSOE-925 with: source: https://nuget.cloudsmith.io/lombiq/open-source-orchard-core-extensions/v3/index.json secrets: diff --git a/.github/workflows/publish-nuget.yml b/.github/workflows/publish-nuget.yml index 44a74afd4..2b2a5e738 100644 --- a/.github/workflows/publish-nuget.yml +++ b/.github/workflows/publish-nuget.yml @@ -9,6 +9,6 @@ jobs: publish-nuget: name: Publish to NuGet if: ${{ !contains(github.ref_name, '-preview.') }} - uses: Lombiq/GitHub-Actions/.github/workflows/publish-nuget.yml@dev + uses: Lombiq/GitHub-Actions/.github/workflows/publish-nuget.yml@issue/OSOE-925 secrets: API_KEY: ${{ secrets.DEFAULT_NUGET_PUBLISH_API_KEY }} diff --git a/.github/workflows/validate-nuget-publish.yml b/.github/workflows/validate-nuget-publish.yml index 9f8979c5d..f0fd6be1e 100644 --- a/.github/workflows/validate-nuget-publish.yml +++ b/.github/workflows/validate-nuget-publish.yml @@ -9,4 +9,4 @@ on: jobs: validate-nuget-publish: name: Validate NuGet Publish - uses: Lombiq/GitHub-Actions/.github/workflows/validate-nuget-publish.yml@dev + uses: Lombiq/GitHub-Actions/.github/workflows/validate-nuget-publish.yml@issue/OSOE-925 diff --git a/Lombiq.Tests.UI.AdminTheme/Constants/FeatureIds.cs b/Lombiq.Tests.UI.AdminTheme/Constants/FeatureIds.cs new file mode 100644 index 000000000..aae0b122b --- /dev/null +++ b/Lombiq.Tests.UI.AdminTheme/Constants/FeatureIds.cs @@ -0,0 +1,6 @@ +namespace Lombiq.Tests.UI.AdminTheme.Constants; + +public static class FeatureIds +{ + public const string Area = "Lombiq.Tests.UI.AdminTheme"; +} diff --git a/Lombiq.Tests.UI.AdminTheme/Constants/ResourceNames.cs b/Lombiq.Tests.UI.AdminTheme/Constants/ResourceNames.cs new file mode 100644 index 000000000..7ca3578e9 --- /dev/null +++ b/Lombiq.Tests.UI.AdminTheme/Constants/ResourceNames.cs @@ -0,0 +1,8 @@ +namespace Lombiq.Tests.UI.AdminTheme.Constants; + +public static class ResourceNames +{ + private const string Prefix = FeatureIds.Area + "."; + + public const string General = Prefix + nameof(General); +} diff --git a/Lombiq.Tests.UI.AdminTheme/Lombiq.Tests.UI.AdminTheme.csproj b/Lombiq.Tests.UI.AdminTheme/Lombiq.Tests.UI.AdminTheme.csproj new file mode 100644 index 000000000..028445814 --- /dev/null +++ b/Lombiq.Tests.UI.AdminTheme/Lombiq.Tests.UI.AdminTheme.csproj @@ -0,0 +1,21 @@ + + + net10.0 + enable + + + + + + + Lombiq UI Testing Toolbox - Admin Theme + 2020 + Adjustments for the stock Orchard Core admin theme to make it more automation-friendly. + OrchardCore;Lombiq;AspNetCore;Selenium;Atata;Shouldly;xUnit;Axe;AccessibilityTesting;UITesting;Testing;Automation + https://github.com/Lombiq/UI-Testing-Toolbox + https://github.com/Lombiq/UI-Testing-Toolbox/tree/dev/Lombiq.Tests.UI.AdminTheme + + + + + diff --git a/Lombiq.Tests.UI.AdminTheme/Manifest.cs b/Lombiq.Tests.UI.AdminTheme/Manifest.cs new file mode 100644 index 000000000..1942fe0c5 --- /dev/null +++ b/Lombiq.Tests.UI.AdminTheme/Manifest.cs @@ -0,0 +1,12 @@ +using OrchardCore.DisplayManagement.Manifest; +using OrchardCore.Modules.Manifest; + +[assembly: Theme( + Name = "Lombiq UI Testing Toolbox - Admin Theme", + Author = "Lombiq Technologies", + Website = "https://github.com/Lombiq/UI-Testing-Toolbox", + Version = "0.0.1", + Description = "Adjustments for the stock Orchard Core admin theme to make it more automation-friendly.", + BaseTheme = "TheAdmin", + Dependencies = ["OrchardCore.Themes", "TheAdmin"], + Tags = [ManifestConstants.AdminTag])] diff --git a/Lombiq.Tests.UI.AdminTheme/NuGetIcon.png b/Lombiq.Tests.UI.AdminTheme/NuGetIcon.png new file mode 100644 index 000000000..162a00508 Binary files /dev/null and b/Lombiq.Tests.UI.AdminTheme/NuGetIcon.png differ diff --git a/Lombiq.Tests.UI.AdminTheme/ResourceManagementOptionsConfiguration.cs b/Lombiq.Tests.UI.AdminTheme/ResourceManagementOptionsConfiguration.cs new file mode 100644 index 000000000..58b598ec6 --- /dev/null +++ b/Lombiq.Tests.UI.AdminTheme/ResourceManagementOptionsConfiguration.cs @@ -0,0 +1,12 @@ +using Lombiq.HelpfulLibraries.OrchardCore.ResourceManagement; +using Lombiq.Tests.UI.AdminTheme.Constants; + +namespace Lombiq.Tests.UI.AdminTheme; + +public class ResourceManagementOptionsConfiguration : ResourceManagementOptionsConfiguratorBase +{ + protected override string Area => FeatureIds.Area; + + protected override void Configure(ResourceManagementContext context) => + context.DefineStyle(ResourceNames.General, "general.css"); +} diff --git a/Lombiq.Tests.UI.AdminTheme/Startup.cs b/Lombiq.Tests.UI.AdminTheme/Startup.cs new file mode 100644 index 000000000..5ec0c47c9 --- /dev/null +++ b/Lombiq.Tests.UI.AdminTheme/Startup.cs @@ -0,0 +1,18 @@ +using Lombiq.Tests.UI.AdminTheme.Constants; +using Microsoft.Extensions.DependencyInjection; +using OrchardCore.Modules; + +namespace Lombiq.Tests.UI.AdminTheme; + +public class Startup : StartupBase +{ + public override void ConfigureServices(IServiceCollection services) + { + services.AddResourceManagementConfiguration(); + services.AddResourceFilter( + builder => builder + .Always() + .RegisterStylesheet(ResourceNames.General), + FeatureIds.Area); + } +} diff --git a/Lombiq.Tests.UI.AdminTheme/wwwroot/Theme.png b/Lombiq.Tests.UI.AdminTheme/wwwroot/Theme.png new file mode 100644 index 000000000..44dfef206 Binary files /dev/null and b/Lombiq.Tests.UI.AdminTheme/wwwroot/Theme.png differ diff --git a/Lombiq.Tests.UI.AdminTheme/wwwroot/css/general.css b/Lombiq.Tests.UI.AdminTheme/wwwroot/css/general.css new file mode 100644 index 000000000..f069d07f2 --- /dev/null +++ b/Lombiq.Tests.UI.AdminTheme/wwwroot/css/general.css @@ -0,0 +1,5 @@ +/* Forcefully remove all CSS animations from the page. This greatly improves screenshot and menu navigation reliability + without compromising the test's authenticity. */ +* { + transition: none !important; /* stylelint-disable-line declaration-no-important */ +} diff --git a/Lombiq.Tests.UI.AppExtensions/CompatibilitySuppressions.xml b/Lombiq.Tests.UI.AppExtensions/CompatibilitySuppressions.xml new file mode 100644 index 000000000..8af156c87 --- /dev/null +++ b/Lombiq.Tests.UI.AppExtensions/CompatibilitySuppressions.xml @@ -0,0 +1,8 @@ + + + + + PKV006 + net8.0 + + \ No newline at end of file diff --git a/Lombiq.Tests.UI.AppExtensions/Lombiq.Tests.UI.AppExtensions.csproj b/Lombiq.Tests.UI.AppExtensions/Lombiq.Tests.UI.AppExtensions.csproj index 5740b73ad..0d55de30b 100644 --- a/Lombiq.Tests.UI.AppExtensions/Lombiq.Tests.UI.AppExtensions.csproj +++ b/Lombiq.Tests.UI.AppExtensions/Lombiq.Tests.UI.AppExtensions.csproj @@ -1,44 +1,28 @@ - - + - net8.0 + net10.0 false - $(DefaultItemExcludes);.git* + + + Lombiq UI Testing Toolbox - App Extensions - Lombiq Technologies - Copyright © 2020, Lombiq Technologies Ltd. - Lombiq UI Testing Toolbox - App Extensions: UI testing-related configuration extensions for the web app under test. See the project website for detailed documentation. - NuGetIcon.png + 2020 + UI testing-related configuration extensions for the web app under test. OrchardCore;Lombiq;AspNetCore;Selenium;Atata;Shouldly;xUnit;Axe;AccessibilityTesting;UITesting;Testing;Automation https://github.com/Lombiq/UI-Testing-Toolbox - - BSD-3-Clause - - - - - - - - - - - - - - - - - + + + + diff --git a/Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj b/Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj index ce75d6766..e525d24e7 100644 --- a/Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj +++ b/Lombiq.Tests.UI.Samples/Lombiq.Tests.UI.Samples.csproj @@ -1,11 +1,12 @@ - - + - net8.0 + net10.0 false - Exe + + + @@ -14,29 +15,10 @@ - - PreserveNewest - - - PreserveNewest - true - PreserveNewest - - - PreserveNewest - - - PreserveNewest - - - - - - - - all - runtime; build; native; contentfiles; analyzers; buildtransitive - + + + + @@ -44,4 +26,6 @@ + + diff --git a/Lombiq.Tests.UI.Samples/Recipes/Lombiq.OSOCE.Tests.Elasticsearch.recipe.json b/Lombiq.Tests.UI.Samples/Recipes/Lombiq.OSOCE.Tests.Elasticsearch.recipe.json index 193f705d5..8c74f36a7 100644 --- a/Lombiq.Tests.UI.Samples/Recipes/Lombiq.OSOCE.Tests.Elasticsearch.recipe.json +++ b/Lombiq.Tests.UI.Samples/Recipes/Lombiq.OSOCE.Tests.Elasticsearch.recipe.json @@ -20,7 +20,7 @@ "name": "feature", "disable": [], "enable": [ - "OrchardCore.Search.Elasticsearch", + "OrchardCore.Elasticsearch", "OrchardCore.Search" ] }, @@ -29,25 +29,119 @@ "Indices": [ { "elasticsearchshouldwork": { - "AnalyzerName": "standard", - "IndexLatest": false, - "IndexedContentTypes": [ - "BlogPost" - ], - "Culture": "any", - "StoreSourceData": true + "Id": "elasticsearchshouldwork000", + "ProviderName": "Elasticsearch", + "Type": "Content", + "CreatedUtc": "2026-01-17T19:21:17Z", + "Properties": { + "ContentIndexMetadata": { + "IndexLatest": false, + "IndexedContentTypes": [ + "BlogPost" + ], + "Culture": "any" + }, + "ElasticsearchIndexMetadata": { + "StoreSourceData": true, + "AnalyzerName": "standard", + "IndexMappings": { + "KeyFieldName": "ContentItemId", + "Mapping": { + "dynamic_templates": [ + { + "*.Inherited": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "*.Inherited" + } + }, + { + "*.Ids": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "*.Ids" + } + }, + { + "*.Location": { + "mapping": { + "type": "geo_point" + }, + "match_mapping_type": "object", + "path_match": "*.Location" + } + } + ], + "properties": { + "ContentItemId": { + "type": "keyword" + }, + "ContentItemVersionId": { + "type": "keyword" + }, + "Content.ContentItem.Owner": { + "type": "keyword" + }, + "Content.ContentItem.FullText": { + "type": "text" + }, + "Content.ContentItem.ContainedPart": { + "properties": { + "Ids": { + "type": "keyword" + }, + "Order": { + "type": "float" + } + }, + "type": "object" + }, + "Content.ContentItem.DisplayText": { + "properties": { + "Analyzed": { + "type": "text" + }, + "Normalized": { + "type": "keyword" + }, + "Keyword": { + "type": "keyword" + } + }, + "type": "object" + }, + "Content.ContentItem.ContentType": { + "type": "keyword" + } + }, + "_source": { + "enabled": true, + "excludes": [ + "Content.ContentItem.DisplayText.Analyzed" + ] + } + } + } + }, + "ElasticsearchDefaultQueryMetadata": { + "QueryAnalyzerName": "standard", + "DefaultSearchFields": [ + "Content.ContentItem.FullText" + ] + } + } } } ] }, { "name": "Settings", - "ElasticSettings": { - "SearchIndex": "elasticsearchshouldwork", - "DefaultSearchFields": [ - "Content.ContentItem.FullText" - ], - "AllowElasticQueryStringQueryInSearch": false + "SearchSettings": { + "DefaultIndexProfileName": "elasticsearchshouldwork" } }, { diff --git a/Lombiq.Tests.UI.Samples/Tests/AzureBlobStorageTests.cs b/Lombiq.Tests.UI.Samples/Tests/AzureBlobStorageTests.cs index afc6362a0..6d67dba6f 100644 --- a/Lombiq.Tests.UI.Samples/Tests/AzureBlobStorageTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/AzureBlobStorageTests.cs @@ -35,8 +35,7 @@ public Task TogglingFeaturesShouldWorkWithAzureBlobStorage() => { configuration.UseAzureBlobStorage = true; - configuration.ResponseLogFilter = e => - e.IsNonSuccessResponseAndNotExpectedNotFoundResponse(ShortcutsUITestContextExtensions.FeatureToggleTestBenchUrl); + configuration.WithIgnoreExpectedNotFoundResponseFilter(ShortcutsUITestContextExtensions.FeatureToggleTestBenchUrl); }); } diff --git a/Lombiq.Tests.UI.Samples/Tests/BasicTests.cs b/Lombiq.Tests.UI.Samples/Tests/BasicTests.cs index bb86b00f2..72aa3cdbe 100644 --- a/Lombiq.Tests.UI.Samples/Tests/BasicTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/BasicTests.cs @@ -77,8 +77,7 @@ public Task TogglingFeaturesShouldWork() => // Note that similarly, you can filter out log entries from the browser log with BrowserLogFilter. You // can also adjust the assertion logic with AssertResponseLog and AssertBrowserLog, but it's best to // filter out entries in the first place. - configuration.ResponseLogFilter = e => - e.IsNonSuccessResponseAndNotExpectedNotFoundResponse(ShortcutsUITestContextExtensions.FeatureToggleTestBenchUrl)); + configuration.WithIgnoreExpectedNotFoundResponseFilter(ShortcutsUITestContextExtensions.FeatureToggleTestBenchUrl)); // Let's see a couple more useful shortcuts in action. [Fact] diff --git a/Lombiq.Tests.UI.Samples/Tests/ErrorHandlingTests.cs b/Lombiq.Tests.UI.Samples/Tests/ErrorHandlingTests.cs index 42eafd88d..4fa3a24ad 100644 --- a/Lombiq.Tests.UI.Samples/Tests/ErrorHandlingTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/ErrorHandlingTests.cs @@ -122,6 +122,7 @@ public Task ErrorDuringSetupShouldHaltTest() => configuration.GitHubActionsOutputConfiguration.EnableErrorAnnotations = false; // We introduce a custom setup operation that has an intentionally invalid SQL Server configuration. + configuration.SetupConfiguration.SetupWithHttpClient = false; configuration.SetupConfiguration.SetupOperation = async context => { await context.GoToSetupAndSetupOrchardCoreAsync( diff --git a/Lombiq.Tests.UI.Samples/Tests/MonkeyTests.cs b/Lombiq.Tests.UI.Samples/Tests/MonkeyTests.cs index 171357a64..661ee5427 100644 --- a/Lombiq.Tests.UI.Samples/Tests/MonkeyTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/MonkeyTests.cs @@ -74,7 +74,7 @@ public Task TestAdminPagesAsMonkeyRecursivelyShouldWorkWithAdminUser() => }, // Requests to /api/graphql without further parameters will fail with HTTP 400, but that's OK, since some // parameters are required. - configuration => configuration.ResponseLogFilter = e => e.IsNonSuccessResponseAndNotExpectedStatusResponse("/api/graphql", 400)); + configuration => configuration.WithIgnoreExpectedStatusResponseFilter("/api/graphql", 400)); // Monkey testing has its own configuration too. Check out the docs of the options too. private static MonkeyTestingOptions CreateMonkeyTestingOptions() => diff --git a/Lombiq.Tests.UI.Samples/Tests/ShiftTimeTests.cs b/Lombiq.Tests.UI.Samples/Tests/ShiftTimeTests.cs index 8282b8d94..160c923e1 100644 --- a/Lombiq.Tests.UI.Samples/Tests/ShiftTimeTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/ShiftTimeTests.cs @@ -35,10 +35,14 @@ public Task TimeShouldUpdate() => await context.GoToAdminRelativeUrlAsync( "/Contents/ContentTypes/LiquidWidget/Create?returnUrl=%2FAdmin%2FLayers&" + "LayerMetadata.Zone=Content&LayerMetadata.Position=1"); + await context.FillInWithRetriesAsync(By.Id("LayerMetadata_Title"), "Current Time Widget"); + await context.SetDropdownByValueAsync(By.Id("LayerMetadata_LayerMetadata_Layer"), "Always"); await context.FillInCodeMirrorEditorWithRetriesAsync( By.CssSelector(".CodeMirror.cm-s-default"), "
{{ \"now\" | utc | date: \"%Y-%m-%d %H:%M\" }}
"); await context.ClickReliablyOnAsync(By.ClassName("publish")); + context.ShouldBeSuccess(); + context.Missing(By.CssSelector(".validation-summary-errors ul li")); var now = await GetNowAsync(context); diff --git a/Lombiq.Tests.UI.Samples/Tests/SqlServerTests.cs b/Lombiq.Tests.UI.Samples/Tests/SqlServerTests.cs index 39a09d372..bdb799213 100644 --- a/Lombiq.Tests.UI.Samples/Tests/SqlServerTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/SqlServerTests.cs @@ -34,8 +34,7 @@ public Task TogglingFeaturesShouldWorkWithSqlServer() => { configuration.UseSqlServer = true; - configuration.ResponseLogFilter = e => - e.IsNonSuccessResponseAndNotExpectedNotFoundResponse(ShortcutsUITestContextExtensions.FeatureToggleTestBenchUrl); + configuration.WithIgnoreExpectedNotFoundResponseFilter(ShortcutsUITestContextExtensions.FeatureToggleTestBenchUrl); }); } diff --git a/Lombiq.Tests.UI.Samples/Tests/TenantTests.cs b/Lombiq.Tests.UI.Samples/Tests/TenantTests.cs index 459a4ac2b..87242e70e 100644 --- a/Lombiq.Tests.UI.Samples/Tests/TenantTests.cs +++ b/Lombiq.Tests.UI.Samples/Tests/TenantTests.cs @@ -31,6 +31,7 @@ public Task CreatingTenantShouldWork() => await context.SignInDirectlyAsync(); // Create the tenant with a custom admin user. + await context.EnableTenantsFeatureDirectlyAsync(); await context.CreateAndSwitchToTenantAsync( TestTenantName, TestTenantUrlPrefix, diff --git a/Lombiq.Tests.UI.Shortcuts/CompatibilitySuppressions.xml b/Lombiq.Tests.UI.Shortcuts/CompatibilitySuppressions.xml new file mode 100644 index 000000000..8af156c87 --- /dev/null +++ b/Lombiq.Tests.UI.Shortcuts/CompatibilitySuppressions.xml @@ -0,0 +1,8 @@ + + + + + PKV006 + net8.0 + + \ No newline at end of file diff --git a/Lombiq.Tests.UI.Shortcuts/Lombiq.Tests.UI.Shortcuts.csproj b/Lombiq.Tests.UI.Shortcuts/Lombiq.Tests.UI.Shortcuts.csproj index 3364d3c4e..45c352cd2 100644 --- a/Lombiq.Tests.UI.Shortcuts/Lombiq.Tests.UI.Shortcuts.csproj +++ b/Lombiq.Tests.UI.Shortcuts/Lombiq.Tests.UI.Shortcuts.csproj @@ -1,52 +1,25 @@ - - + - net8.0 + net10.0 false - true - $(DefaultItemExcludes);.git* + + + Lombiq UI Testing Toolbox - Shortcuts - Lombiq Technologies - Copyright © 2020, Lombiq Technologies Ltd. - Lombiq UI Testing Toolbox - Shortcuts: Provides some useful shortcuts for common operations that UI tests might want to do or check, e.g. turning features on or off, or logging in users. See the project website for detailed documentation. + 2020 + Provides some useful shortcuts for common operations that UI tests might want to do or check, e.g. turning features on or off, or logging in users. OrchardCore;Lombiq;AspNetCore;Selenium;Atata;Shouldly;xUnit;Axe;AccessibilityTesting;UITesting;Testing;Automation - NuGetIcon.png https://github.com/Lombiq/UI-Testing-Toolbox https://github.com/Lombiq/UI-Testing-Toolbox/tree/dev/Lombiq.Tests.UI.Shortcuts - BSD-3-Clause - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + diff --git a/Lombiq.Tests.UI.Shortcuts/Manifest.cs b/Lombiq.Tests.UI.Shortcuts/Manifest.cs index 706a3a091..ac2c43828 100644 --- a/Lombiq.Tests.UI.Shortcuts/Manifest.cs +++ b/Lombiq.Tests.UI.Shortcuts/Manifest.cs @@ -53,10 +53,10 @@ )] [assembly: Feature( - Id = Swagger, - Name = "Swagger - Shortcuts - Lombiq UI Testing Toolbox", + Id = OpenApi, + Name = "OpenAPI - Shortcuts - Lombiq UI Testing Toolbox", Category = "Development", - Description = DescriptionUiTestWarning + "Provides a Swagger endpoint to generate a JSON OpenAPI definition for " + + Description = DescriptionUiTestWarning + "Provides an endpoint to generate a JSON OpenAPI definition for " + "the web APIs available in the app. Used in security scanning." )] diff --git a/Lombiq.Tests.UI.Shortcuts/ShortcutsFeatureIds.cs b/Lombiq.Tests.UI.Shortcuts/ShortcutsFeatureIds.cs index 1f7532430..0915a78ec 100644 --- a/Lombiq.Tests.UI.Shortcuts/ShortcutsFeatureIds.cs +++ b/Lombiq.Tests.UI.Shortcuts/ShortcutsFeatureIds.cs @@ -10,6 +10,6 @@ public static class ShortcutsFeatureIds public const string FeatureToggleTestBench = $"{Default}.{nameof(FeatureToggleTestBench)}"; public const string MediaCachePurge = $"{Default}.{nameof(MediaCachePurge)}"; public const string ShiftTime = $"{Default}.{nameof(ShiftTime)}"; - public const string Swagger = $"{Default}.{nameof(Swagger)}"; + public const string OpenApi = $"{Default}.{nameof(OpenApi)}"; public const string Workflows = $"{Default}.{nameof(Workflows)}"; } diff --git a/Lombiq.Tests.UI.Shortcuts/Startup.cs b/Lombiq.Tests.UI.Shortcuts/Startup.cs index 4b75d20ef..708418947 100644 --- a/Lombiq.Tests.UI.Shortcuts/Startup.cs +++ b/Lombiq.Tests.UI.Shortcuts/Startup.cs @@ -27,15 +27,14 @@ public override void Configure(IApplicationBuilder app, IEndpointRouteBuilder ro app.UseMiddleware(); } -[Feature(ShortcutsFeatureIds.Swagger)] -public sealed class SwaggerStartup : StartupBase +[Feature(ShortcutsFeatureIds.OpenApi)] +public sealed class OpenApiSetup : StartupBase { public override void ConfigureServices(IServiceCollection services) => - services.AddSwaggerGen(swaggerGenOptions => - swaggerGenOptions.SwaggerDoc("v1", new OpenApiInfo { Title = "Orchard Core API", Version = "v1" })); + services.AddOpenApi(options => options.OpenApiVersion = OpenApiSpecVersion.OpenApi3_1); public override void Configure(IApplicationBuilder app, IEndpointRouteBuilder routes, IServiceProvider serviceProvider) => - app.UseSwagger(); + routes.MapOpenApi(); } [Feature(ShortcutsFeatureIds.ShiftTime)] diff --git a/Lombiq.Tests.UI.TestingModule/Controllers/SqlQueryMonitoringScenarioController.cs b/Lombiq.Tests.UI.TestingModule/Controllers/SqlQueryMonitoringScenarioController.cs index 932856c73..de15794e8 100644 --- a/Lombiq.Tests.UI.TestingModule/Controllers/SqlQueryMonitoringScenarioController.cs +++ b/Lombiq.Tests.UI.TestingModule/Controllers/SqlQueryMonitoringScenarioController.cs @@ -34,7 +34,7 @@ public SqlQueryMonitoringScenarioController( /// public async Task Index() { - var contentItemCount = await _session.QueryIndex().CountAsync(); + var contentItemCount = await _session.QueryIndex().CountAsync(HttpContext.RequestAborted); return View(model: contentItemCount); } @@ -43,7 +43,7 @@ public async Task Index() /// public async Task AsyncQuery() { - var contentItemCount = await _session.QueryIndex().CountAsync(); + var contentItemCount = await _session.QueryIndex().CountAsync(HttpContext.RequestAborted); return Ok(contentItemCount); } @@ -74,7 +74,7 @@ public async Task RawExecuteNonQuery() public async Task CustomSessionQuery() { await using var session = _store.CreateSession(); - var contentItemCount = await session.QueryIndex().CountAsync(); + var contentItemCount = await session.QueryIndex().CountAsync(HttpContext.RequestAborted); return Ok(contentItemCount); } diff --git a/Lombiq.Tests.UI.TestingModule/Lombiq.Tests.UI.TestingModule.csproj b/Lombiq.Tests.UI.TestingModule/Lombiq.Tests.UI.TestingModule.csproj index 6d810cbc9..73fe15989 100644 --- a/Lombiq.Tests.UI.TestingModule/Lombiq.Tests.UI.TestingModule.csproj +++ b/Lombiq.Tests.UI.TestingModule/Lombiq.Tests.UI.TestingModule.csproj @@ -1,23 +1,12 @@ - - + - net8.0 + net10.0 false - true - $(DefaultItemExcludes);.git* - - - - - - - - - - - - + + + + diff --git a/Lombiq.Tests.UI.Tests.UI/Lombiq.Tests.UI.Tests.UI.csproj b/Lombiq.Tests.UI.Tests.UI/Lombiq.Tests.UI.Tests.UI.csproj index 15b6f08ad..23e3f0e99 100644 --- a/Lombiq.Tests.UI.Tests.UI/Lombiq.Tests.UI.Tests.UI.csproj +++ b/Lombiq.Tests.UI.Tests.UI/Lombiq.Tests.UI.Tests.UI.csproj @@ -1,7 +1,7 @@ - net8.0 + net10.0 enable enable diff --git a/Lombiq.Tests.UI/.config/dotnet-tools.json b/Lombiq.Tests.UI/.config/dotnet-tools.json index 838b1f583..18a4ad588 100644 --- a/Lombiq.Tests.UI/.config/dotnet-tools.json +++ b/Lombiq.Tests.UI/.config/dotnet-tools.json @@ -3,7 +3,7 @@ "isRoot": true, "tools": { "rnwood.smtp4dev": { - "version": "3.14.0", + "version": "3.15.0", "commands": [ "smtp4dev" ] diff --git a/Lombiq.Tests.UI/Attributes/AllBrowsersAttribute.cs b/Lombiq.Tests.UI/Attributes/AllBrowsersAttribute.cs index d45d6056a..627ab5472 100644 --- a/Lombiq.Tests.UI/Attributes/AllBrowsersAttribute.cs +++ b/Lombiq.Tests.UI/Attributes/AllBrowsersAttribute.cs @@ -14,7 +14,7 @@ public sealed class AllBrowsersAttribute : DataAttribute { public override ValueTask> GetData(MethodInfo testMethod, DisposalTracker disposalTracker) { - var browsers = (IEnumerable)Enum.GetValues(typeof(Browser)); + var browsers = Enum.GetValues(); var dataRows = new List(); foreach (var browser in browsers) diff --git a/Lombiq.Tests.UI/BasicOrchardFeaturesTesting/BasicFeaturesTestingUITestContextExtensions.cs b/Lombiq.Tests.UI/BasicOrchardFeaturesTesting/BasicFeaturesTestingUITestContextExtensions.cs index ab4680246..a8ef6deb1 100644 --- a/Lombiq.Tests.UI/BasicOrchardFeaturesTesting/BasicFeaturesTestingUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/BasicOrchardFeaturesTesting/BasicFeaturesTestingUITestContextExtensions.cs @@ -318,8 +318,11 @@ private static Task TestSetupAsync( OrchardCoreSetupParameters setupParameters, string testName, bool shouldBeSuccess) => - context.ExecuteTestAsync(testName, () => context - .GoToSetupAndSetupOrchardCoreAsync(setupParameters, shouldBeSuccess)); + context.ExecuteTestAsync(testName, () => + { + context.Configuration.SetupConfiguration.SetupWithHttpClient = false; + return context.GoToSetupAndSetupOrchardCoreAsync(setupParameters, shouldBeSuccess); + }); private static Task TestLoginAsync( this UITestContext context, diff --git a/Lombiq.Tests.UI/BasicOrchardFeaturesTesting/MediaOperationsTestingUITestContextExtensions.cs b/Lombiq.Tests.UI/BasicOrchardFeaturesTesting/MediaOperationsTestingUITestContextExtensions.cs index bdff7c5cb..7a9d265ea 100644 --- a/Lombiq.Tests.UI/BasicOrchardFeaturesTesting/MediaOperationsTestingUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/BasicOrchardFeaturesTesting/MediaOperationsTestingUITestContextExtensions.cs @@ -30,6 +30,9 @@ public static Task TestMediaOperationsAsync(this UITestContext context) => context.WaitForPageLoad(); await context.ClickReliablyOnAsync(By.CssSelector("body")); + // Check for errors before validating the result to aim for most useful information in case of errors. + await context.AssertLogsAsync(); + context.Missing(By.CssSelector("#mediaContainerMain .upload-list .text-danger")); context.Exists(By.XPath($"//span[contains(text(), '{imageName}')]")); await context diff --git a/Lombiq.Tests.UI/CompatibilitySuppressions.xml b/Lombiq.Tests.UI/CompatibilitySuppressions.xml new file mode 100644 index 000000000..8af156c87 --- /dev/null +++ b/Lombiq.Tests.UI/CompatibilitySuppressions.xml @@ -0,0 +1,8 @@ + + + + + PKV006 + net8.0 + + \ No newline at end of file diff --git a/Lombiq.Tests.UI/Constants/DirectoryPaths.cs b/Lombiq.Tests.UI/Constants/DirectoryPaths.cs index aa8d0cf99..62c077d0e 100644 --- a/Lombiq.Tests.UI/Constants/DirectoryPaths.cs +++ b/Lombiq.Tests.UI/Constants/DirectoryPaths.cs @@ -11,14 +11,13 @@ public static class DirectoryPaths public const string Screenshots = nameof(Screenshots); public const string Downloads = nameof(Downloads); + [Obsolete($"Use {nameof(UITestContext.TempDirectoryPath)} or {nameof(UITestContext.GetTempSubDirectoryPath)}() " + + $"in {nameof(UITestContext)} instead.")] public static string GetTempDirectoryPath(params string[] subDirectoryNames) => Path.Combine([Environment.CurrentDirectory, Temp, .. subDirectoryNames]); - [Obsolete($"Use {nameof(UITestContext)}.{nameof(UITestContext.GetTempSubDirectoryPath)}() instead.")] - public static string GetTempSubDirectoryPath(string contextId, params string[] subDirectoryNames) => - GetTempDirectoryPath([contextId, .. subDirectoryNames]); - - [Obsolete($"Use {nameof(UITestContext)}.{nameof(UITestContext.ScreenshotsDirectoryPath)} instead.")] - public static string GetScreenshotsDirectoryPath(string contextId) => - GetTempSubDirectoryPath(contextId, Screenshots); + internal static string GetTempDirectoryPathWithFallback(string path) => + string.IsNullOrEmpty(path) + ? Path.Combine(Environment.CurrentDirectory, Temp) + : path; } diff --git a/Lombiq.Tests.UI/Docs/Configuration.md b/Lombiq.Tests.UI/Docs/Configuration.md index 3948a83da..710f149bb 100644 --- a/Lombiq.Tests.UI/Docs/Configuration.md +++ b/Lombiq.Tests.UI/Docs/Configuration.md @@ -24,7 +24,7 @@ Note also that some projects' _xunit.runner.json_ files may include the flag [`s Certain test execution parameters can be configured externally too, the ones retrieved via the `TestConfigurationManager` class. All configuration options are basic key-value pairs and can be provided in one of the two ways: -- Key-value pairs in a _TestConfiguration.json_ file. Note that this file needs to be in the folder where the UI tests execute. By default this is the build output folder of the given test project, i.e. where the projects's DLL is generated (e.g. _bin/Debug/net6.0_). +- Key-value pairs in a _TestConfiguration.json_ file. Note that this file needs to be in the folder where the UI tests execute. By default this is the build output folder of the given test project, i.e. where the projects's DLL is generated (e.g. _bin/Debug/net10.0_). - Environment variables: Their names should be prefixed with `Lombiq_Tests_UI`, followed by the config with a `__` as it is with [(ASP).NET configuration](https://learn.microsoft.com/en-us/aspnet/core/fundamentals/configuration/), e.g. `Lombiq_Tests_UI__OrchardCoreUITestExecutorConfiguration__MaxRetryCount` (instead of the double underscore you can also use a `:` on certain platforms like Windows). Keep in mind that you can set these just for the current session too. Configuration in environment variables will take precedence over the _TestConfiguration.json_ file. When you're setting environment variables while trying out test execution keep in mind that you'll have to restart the app after changing any environment variable. Here's a full _TestConfiguration.json_ file example, something appropriate during development when you have a fast machine (probably faster then the one used to execute these tests) and want tests to fail fast instead of being reliable: diff --git a/Lombiq.Tests.UI/Docs/Troubleshooting.md b/Lombiq.Tests.UI/Docs/Troubleshooting.md index c854f72f4..16c8befa6 100644 --- a/Lombiq.Tests.UI/Docs/Troubleshooting.md +++ b/Lombiq.Tests.UI/Docs/Troubleshooting.md @@ -2,7 +2,7 @@ ## General tips -- When a test fails it'll create a dump in the test execution's folder (usually something like _bin/Debug/net8.0_ under your test project), in a new _TestDumps_ folder. (Though succeeding tests may also provide some output files there.) This should help you pinpoint where the issue is even if the test was run in a CI environment, and you can't reproduce it locally. The dump contains the following: +- When a test fails it'll create a dump in the test execution's folder (usually something like _bin/Debug/net10.0_ under your test project), in a new _TestDumps_ folder. (Though succeeding tests may also provide some output files there.) This should help you pinpoint where the issue is even if the test was run in a CI environment, and you can't reproduce it locally. The dump contains the following: - The Orchard application's folder, including settings files, the SQLite or SQL Server DB, logs, etc. that you can utilize to see log entries and to run the app from that state. - Browser logs, i.e. the developer console output. - Screenshots of each page in order the test visited them, as well as when the test failed (Windows Photo Viewer won't be able to open it though, use something else like the Windows 10 Photos app). diff --git a/Lombiq.Tests.UI/Exceptions/AccessibilityAssertionException.cs b/Lombiq.Tests.UI/Exceptions/AccessibilityAssertionException.cs index ca4de30f8..8e19a0683 100644 --- a/Lombiq.Tests.UI/Exceptions/AccessibilityAssertionException.cs +++ b/Lombiq.Tests.UI/Exceptions/AccessibilityAssertionException.cs @@ -1,4 +1,5 @@ using Deque.AxeCore.Commons; +using Lombiq.Tests.UI.Models; using System; namespace Lombiq.Tests.UI.Exceptions; @@ -6,13 +7,18 @@ namespace Lombiq.Tests.UI.Exceptions; public class AccessibilityAssertionException : Exception, IAssertionException { public AxeResult AxeResult { get; } + public AccessibilityCheckingResult Result { get; } public AccessibilityAssertionException(AxeResult axeResult, bool createReportOnFailure, Exception innerException) + : this((AccessibilityCheckingResult)axeResult, createReportOnFailure, innerException) => + AxeResult = axeResult; + + public AccessibilityAssertionException(AccessibilityCheckingResult result, bool createReportOnFailure, Exception innerException) : base( "Asserting the accessibility analysis result failed." + (createReportOnFailure ? " Check the accessibility report failure dump for details." : string.Empty), innerException) => - AxeResult = axeResult; + Result = result; public AccessibilityAssertionException() { diff --git a/Lombiq.Tests.UI/Extensions/AccessibilityCheckingOrchardCoreUITestExecutorConfigurationExtensions.cs b/Lombiq.Tests.UI/Extensions/AccessibilityCheckingOrchardCoreUITestExecutorConfigurationExtensions.cs index 9506688b1..1a8f67e4e 100644 --- a/Lombiq.Tests.UI/Extensions/AccessibilityCheckingOrchardCoreUITestExecutorConfigurationExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/AccessibilityCheckingOrchardCoreUITestExecutorConfigurationExtensions.cs @@ -1,6 +1,8 @@ using Deque.AxeCore.Commons; using Deque.AxeCore.Selenium; +using Lombiq.Tests.UI.Models; using Lombiq.Tests.UI.Services; +using Shouldly; using System; using System.Threading.Tasks; @@ -23,7 +25,7 @@ public static class AccessibilityCheckingOrchardCoreUITestExecutorConfigurationE public static void SetUpAccessibilityCheckingAssertionOnPageChange( this OrchardCoreUITestExecutorConfiguration configuration, Action axeBuilderConfigurator = null, - Action assertAxeResult = null) + Action assertAxeResult = null) { if (!configuration.CustomConfiguration.TryAdd("AccessibilityCheckingAssertionOnPageChangeWasSetUp", value: true)) return; @@ -37,4 +39,62 @@ public static void SetUpAccessibilityCheckingAssertionOnPageChange( return Task.CompletedTask; }; } + + /// + /// Shortcut for adding a filter to 's . + /// + public static void WithAxeIncompletesFilter( + this OrchardCoreUITestExecutorConfiguration configuration, + string name, + Func filter) => + configuration.AccessibilityCheckingConfiguration.AxeResultIncompleteFilters[name] = filter; + + /// + /// Shortcut for adding a filter to 's . + /// + public static void WithAxeIncompletesFilter( + this OrchardCoreUITestExecutorConfiguration configuration, + string name, + string idToExclude) => + configuration.WithAxeIncompletesFilter(name, item => item.Id != idToExclude); + + /// + /// Shortcut for adding a filter to 's . + /// + public static void WithAxeViolationsFilters( + this OrchardCoreUITestExecutorConfiguration configuration, + string name, + Func filter) => + configuration.AccessibilityCheckingConfiguration.AxeResultViolationsFilters.Add(name, filter); + + /// + /// Shortcut for adding a filter to 's . + /// + public static void WithAxeViolationsFilters( + this OrchardCoreUITestExecutorConfiguration configuration, + string name, + string idToExclude) => + configuration.WithAxeViolationsFilters(name, item => item.Id != idToExclude); + + /// + /// Adds exceptions for color contrast accessibility violations by selector. + /// + public static void WithAxeColorContrastViolationsFilters( + this OrchardCoreUITestExecutorConfiguration configuration, + params string[] selectors) + { + selectors.ShouldNotBeEmpty(); + configuration.WithAxeViolationsFilters( + $"{nameof(WithAxeColorContrastViolationsFilters)}: \"{string.Join("\", \"", selectors)}\"", + item => !(string.Equals(item.Id, "color-contrast", StringComparison.Ordinal) && + item.Nodes.TrueForAll(node => selectors.Exists(selector => node.Target.Selector.Contains(selector))))); + } } diff --git a/Lombiq.Tests.UI/Extensions/AccessibilityCheckingUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/AccessibilityCheckingUITestContextExtensions.cs index 6cd6bf04e..914091091 100644 --- a/Lombiq.Tests.UI/Extensions/AccessibilityCheckingUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/AccessibilityCheckingUITestContextExtensions.cs @@ -3,8 +3,10 @@ using Lombiq.Tests.UI.Constants; using Lombiq.Tests.UI.Exceptions; using Lombiq.Tests.UI.Helpers; +using Lombiq.Tests.UI.Models; using Lombiq.Tests.UI.Services; using System; +using System.Collections.Generic; using System.IO; using TWP.Selenium.Axe.Html; @@ -27,14 +29,21 @@ public static class AccessibilityCheckingUITestContextExtensions public static void AssertAccessibility( this UITestContext context, Action axeBuilderConfigurator = null, - Action assertAxeResult = null) + Action assertAxeResult = null) { var axeResult = context.AnalyzeAccessibility(axeBuilderConfigurator); + var result = (AccessibilityCheckingResult)axeResult; var accessibilityConfiguration = context.Configuration.AccessibilityCheckingConfiguration; try { - (assertAxeResult ?? accessibilityConfiguration.AssertAxeResult)?.Invoke(axeResult); + if (accessibilityConfiguration.AxeResultIncompleteFilters.Count > 0 || + accessibilityConfiguration.AxeResultViolationsFilters.Count > 0) + { + result = FilterAccessibilityResults(result, accessibilityConfiguration); + } + + (assertAxeResult ?? accessibilityConfiguration.AssertAxeResult)?.Invoke(result); } catch (Exception ex) { @@ -57,6 +66,23 @@ public static void AssertAccessibility( } } + private static AccessibilityCheckingResult FilterAccessibilityResults( + AccessibilityCheckingResult axeResult, + AccessibilityCheckingConfiguration accessibilityConfiguration) + { + foreach (var filter in accessibilityConfiguration.AxeResultIncompleteFilters.Values) + { + axeResult.Incomplete.RemoveAll(item => item is null || !filter(item)); + } + + foreach (var filter in accessibilityConfiguration.AxeResultViolationsFilters.Values) + { + axeResult.Violations.RemoveAll(item => item is null || !filter(item)); + } + + return axeResult; + } + /// /// Runs an axe accessibility analysis. Note that you need to run this after every page load, it won't accumulate /// during a session. diff --git a/Lombiq.Tests.UI/Extensions/AxeResultItemExtensions.cs b/Lombiq.Tests.UI/Extensions/AxeResultItemExtensions.cs index adfd68179..e83cc66f9 100644 --- a/Lombiq.Tests.UI/Extensions/AxeResultItemExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/AxeResultItemExtensions.cs @@ -1,7 +1,10 @@ +#nullable enable + using Deque.AxeCore.Commons; using Lombiq.Tests.UI.Services; using Shouldly; using System.Collections.Generic; +using System.Linq; namespace Lombiq.Tests.UI.Extensions; @@ -11,6 +14,12 @@ public static class AxeResultItemExtensions /// Asserts if is empty, and if not then produces an error with s converted into human-readable strings. /// - public static void AxeResultItemsShouldBeEmpty(this IEnumerable axeResultItems) => - axeResultItems.ShouldBeEmpty(AccessibilityCheckingConfiguration.AxeResultItemsToString(axeResultItems)); + public static void AxeResultItemsShouldBeEmpty(this IEnumerable axeResultItems) + { + var results = axeResultItems + .CastWhere() + .ToList(); + + results.ShouldBeEmpty(AccessibilityCheckingConfiguration.AxeResultItemsToString(results)); + } } diff --git a/Lombiq.Tests.UI/Extensions/FakeBrowserVideoSourceExtensions.cs b/Lombiq.Tests.UI/Extensions/FakeBrowserVideoSourceExtensions.cs index 829a16195..14a92d036 100644 --- a/Lombiq.Tests.UI/Extensions/FakeBrowserVideoSourceExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/FakeBrowserVideoSourceExtensions.cs @@ -1,5 +1,5 @@ -using Lombiq.Tests.UI.Constants; using Lombiq.Tests.UI.Models; +using Lombiq.Tests.UI.Services; using System; using System.IO; @@ -7,11 +7,15 @@ namespace Lombiq.Tests.UI.Extensions; public static class FakeBrowserVideoSourceExtensions { - public static string SaveVideoToTempFolder(this FakeBrowserVideoSource source) + [Obsolete("Use the overload that specifies the Temp directory path instead.")] + public static string SaveVideoToTempFolder(this FakeBrowserVideoSource source) => + source.SaveVideoToTempFolder(tempDirectoryPath: null); + + public static string SaveVideoToTempFolder(this FakeBrowserVideoSource source, string tempDirectoryPath) { using var fakeCameraSource = source.StreamProvider(); var fakeCameraSourcePath = Path.ChangeExtension( - DirectoryPaths.GetTempDirectoryPath(Guid.NewGuid().ToString()), + OrchardCoreUITestExecutorConfiguration.GetTempDirectoryPathWithFallback(tempDirectoryPath, Guid.NewGuid().ToString()), GetExtension(source.Format)); using var fakeCameraSourceFile = new FileStream(fakeCameraSourcePath, FileMode.CreateNew, FileAccess.Write); diff --git a/Lombiq.Tests.UI/Extensions/HtmlValidationErrorExtensions.cs b/Lombiq.Tests.UI/Extensions/HtmlValidationErrorExtensions.cs new file mode 100644 index 000000000..7c6201455 --- /dev/null +++ b/Lombiq.Tests.UI/Extensions/HtmlValidationErrorExtensions.cs @@ -0,0 +1,25 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Lombiq.Tests.UI.Models; + +public static class HtmlValidationErrorExtensions +{ + /// + /// Remove entries from if they return when passed into any of the + /// . + /// + internal static IList RemoveIfFalse( + this IList errors, + IEnumerable> filters) + { + foreach (var filter in filters.Where(filter => filter != null)) + { + errors.RemoveAll(error => !filter(error)); + if (errors.Count == 0) return errors; + } + + return errors; + } +} diff --git a/Lombiq.Tests.UI/Extensions/HtmlValidationOrchardCoreUITestExecutorConfigurationExtensions.cs b/Lombiq.Tests.UI/Extensions/HtmlValidationOrchardCoreUITestExecutorConfigurationExtensions.cs index a2314c540..e02eaba68 100644 --- a/Lombiq.Tests.UI/Extensions/HtmlValidationOrchardCoreUITestExecutorConfigurationExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/HtmlValidationOrchardCoreUITestExecutorConfigurationExtensions.cs @@ -1,6 +1,8 @@ using Atata.HtmlValidation; +using Lombiq.Tests.UI.Models; using Lombiq.Tests.UI.Services; using System; +using System.Collections.Generic; using System.Threading.Tasks; namespace Lombiq.Tests.UI.Extensions; @@ -21,7 +23,7 @@ public static class HtmlValidationOrchardCoreUITestExecutorConfigurationExtensio public static void SetUpHtmlValidationAssertionOnPageChange( this OrchardCoreUITestExecutorConfiguration configuration, Action htmlValidationOptionsAdjuster = null, - Func assertHtmlValidationResultAsync = null) + Func, Task> assertHtmlValidationResultAsync = null) { if (!configuration.CustomConfiguration.TryAdd("HtmlValidationAssertionOnPageChangeWasSetUp", value: true)) return; diff --git a/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs b/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs index 0ad3f56cb..aa2c4aef5 100644 --- a/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/HtmlValidationResultExtensions.cs @@ -31,9 +31,9 @@ public static async Task> GetErrorsAsync(this HtmlValidation /// Gets the parsed errors from the HTML validation result. /// Can only be used if the output formatter is set to JSON. /// - public static IEnumerable GetParsedErrors(this HtmlValidationResult result) => ParseOutput(result.Output); + public static IEnumerable GetParsedErrors(this HtmlValidationResult result) => ParseOutput(result.Output); - public static string GetParsedErrorMessageString(IEnumerable errors) => + public static string GetParsedErrorMessageString(IEnumerable errors) => string.Join( '\n', errors.Select(error => @@ -41,7 +41,7 @@ public static string GetParsedErrorMessageString(IEnumerable ParseOutput(string output) + private static IEnumerable ParseOutput(string output) { try { @@ -60,7 +60,7 @@ private static IEnumerable ParseOutput(string output) .Select(message => { var rawMessageText = message.GetRawText(); - return JsonSerializer.Deserialize(rawMessageText); + return JsonSerializer.Deserialize(rawMessageText); }); } catch (JsonException exception) diff --git a/Lombiq.Tests.UI/Extensions/HtmlValidationUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/HtmlValidationUITestContextExtensions.cs index 02c9abca4..c237b6bfa 100644 --- a/Lombiq.Tests.UI/Extensions/HtmlValidationUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/HtmlValidationUITestContextExtensions.cs @@ -1,8 +1,12 @@ using Atata.Cli; using Atata.HtmlValidation; using Lombiq.Tests.UI.Exceptions; +using Lombiq.Tests.UI.Models; using Lombiq.Tests.UI.Services; +using Shouldly; using System; +using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; namespace Lombiq.Tests.UI.Extensions; @@ -23,16 +27,36 @@ public static class HtmlValidationUITestContextExtensions public static async Task AssertHtmlValidityAsync( this UITestContext context, Action htmlValidationOptionsAdjuster = null, - Func assertHtmlValidationResultAsync = null) + Func, Task> assertHtmlValidationResultAsync = null) { - var validationResult = await context.ValidateHtmlAsync(htmlValidationOptionsAdjuster); var validationConfiguration = context.Configuration.HtmlValidationConfiguration; + var validationResult = await context.ValidateHtmlAsync(htmlValidationOptionsAdjuster); + if (validationResult?.GetParsedErrors()?.AsList() is not { Count: > 0 } errors) return; + + var filters = validationConfiguration.HtmlValidationFilters; + assertHtmlValidationResultAsync ??= validationConfiguration.AssertHtmlValidationResultAsync; + assertHtmlValidationResultAsync ??= errors => + { + var humanReadableErrors = HtmlValidationResultExtensions.GetParsedErrorMessageString(errors); + var filtersUsedMessage = $"The following {nameof(HtmlValidationConfiguration.HtmlValidationFilters)} were " + + $"used: {string.Join(", ", filters.Keys)}"; + + errors.ShouldBeEmpty(filters.Count > 0 ? $"{humanReadableErrors}\n\n{filtersUsedMessage}" : humanReadableErrors); + return Task.CompletedTask; + }; try { - var assertTask = (assertHtmlValidationResultAsync ?? validationConfiguration.AssertHtmlValidationResultAsync)? - .Invoke(validationResult); - await (assertTask ?? Task.CompletedTask); + foreach (var filter in filters.Values.Where(filter => filter != null)) + { + errors.RemoveAll(error => !filter(error)); + if (errors.Count == 0) return; + } + + if (assertHtmlValidationResultAsync(errors) is { } assertTask) + { + await assertTask; + } } catch (Exception exception) { diff --git a/Lombiq.Tests.UI/Extensions/MonkeyTestingUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/MonkeyTestingUITestContextExtensions.cs index 6c9d5ce7d..7769bfb3c 100644 --- a/Lombiq.Tests.UI/Extensions/MonkeyTestingUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/MonkeyTestingUITestContextExtensions.cs @@ -2,6 +2,7 @@ using Lombiq.Tests.UI.MonkeyTesting; using Lombiq.Tests.UI.MonkeyTesting.UrlFilters; using Lombiq.Tests.UI.Services; +using OpenQA.Selenium.BiDi.Log; using System.Threading.Tasks; namespace Lombiq.Tests.UI.Extensions; @@ -107,6 +108,9 @@ public static async Task TestFrontendAuthenticatedAndAnonymouslyAsMonkeyRecursiv string signInDirectlyWithUserName = DefaultUser.UserName, string startingRelativeUrl = "/") { + context.Configuration.BrowserLogFilters["Exclude Gremlin info logs"] = entry => + !(entry.Level == Level.Info && entry.Text?.Contains("gremlin") == true); + await TestFrontendAuthenticatedAsMonkeyRecursivelyAsync( context, options, diff --git a/Lombiq.Tests.UI/Extensions/NavigationUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/NavigationUITestContextExtensions.cs index 115e418e5..24c8cda9a 100644 --- a/Lombiq.Tests.UI/Extensions/NavigationUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/NavigationUITestContextExtensions.cs @@ -1,3 +1,5 @@ +using AngleSharp.Dom; +using AngleSharp.Html.Parser; using Atata; using Lombiq.HelpfulLibraries.OrchardCore.Mvc; using Lombiq.Tests.UI.Constants; @@ -7,9 +9,14 @@ using OpenQA.Selenium; using OpenQA.Selenium.Interactions; using OpenQA.Selenium.Support.UI; +using OrchardCore.Environment.Shell; +using Shouldly; using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; using System.Threading.Tasks; - +using Xunit; #pragma warning disable CS0618 // Type or member is obsolete. These are only used in obsolete extension methods. using OrchardCoreContentItemsPage = Lombiq.Tests.UI.Pages.OrchardCoreContentItemsPage; using OrchardCoreDashboardPage = Lombiq.Tests.UI.Pages.OrchardCoreDashboardPage; @@ -250,6 +257,12 @@ public static async Task GoToSetupAndSetupOrchardCoreAsync( OrchardCoreSetupParameters parameters = null, bool shouldBeSuccess = true) { + if (context.Configuration.SetupConfiguration.SetupWithHttpClient && + context.TenantName == ShellSettings.DefaultShellName) + { + return await context.SetupOrchardCoreWithHttpClientAsync(parameters, shouldBeSuccess); + } + parameters ??= new(context); if (!parameters.RunSetupOnCurrentPage) @@ -263,6 +276,66 @@ public static async Task GoToSetupAndSetupOrchardCoreAsync( return new(context.Driver.Url); } + public static async Task SetupOrchardCoreWithHttpClientAsync( + this UITestContext context, + OrchardCoreSetupParameters parameters = null, + bool shouldBeSuccess = true) + { + static async Task GetElementAsync(HttpResponseMessage response, string query) + { + using (response) + { + await using var stream = await response.Content.ReadAsStreamAsync(TestContext.Current.CancellationToken); + var document = await new HtmlParser().ParseDocumentAsync(stream, TestContext.Current.CancellationToken); + return string.IsNullOrEmpty(query) ? document.DocumentElement : document.QuerySelector(query); + } + } + + parameters ??= new(context); + + var cancellationToken = context.Configuration.TestCancellationToken; + + var uri = parameters.RunSetupOnCurrentPage + ? context.GetCurrentUri() + : parameters.SetupUri ?? context.TestStartUri; + + using var client = context.Configuration.BrowserConfiguration.Browser == Browser.None + ? context.CreateHttpClient() + : context.CreateHttpClientWithBrowserContext(); + + var verificationToken = (await GetElementAsync( + await client.GetAsync(uri, cancellationToken), + "input[name=__RequestVerificationToken]"))? + .GetAttribute("value"); + + var formData = new Dictionary + { + ["SiteName"] = parameters.SiteName, + ["RecipeName"] = parameters.RecipeId, + ["SiteTimeZone"] = parameters.SiteTimeZoneValue, + ["DatabaseProvider"] = parameters.DatabaseProvider.ToString(), + ["TablePrefix"] = parameters.TablePrefix, + ["ConnectionString"] = parameters.ConnectionString, + ["Schema"] = string.Empty, + ["UserName"] = parameters.UserName, + ["Email"] = parameters.Email, + ["Password"] = parameters.Password, + ["PasswordConfirmation"] = parameters.Password, + ["Secret"] = string.Empty, + ["__RequestVerificationToken"] = verificationToken, + }; + + using var formContent = new FormUrlEncodedContent(formData); + var responseSetupScript = await GetElementAsync( + await client.PostAsync(uri, formContent, cancellationToken), + "script[src*='OrchardCore.Setup']"); + + (responseSetupScript == null).ShouldBe(shouldBeSuccess, shouldBeSuccess ? "Setup did not succeed." : "Setup succeeded when it shouldn't."); + + await context.GoToAbsoluteUrlAsync(uri, onlyIfNotAlreadyThere: false); + return uri; + } + [Obsolete($"Methods using Page<> classes will be removed in the next version. Use {nameof(GoToRegistrationAsync)} instead.")] public static Task GoToRegistrationPageAsync(this UITestContext context) => context.GoToPageAsync(); @@ -456,6 +529,34 @@ public static Task ClickReliablyOnUntilUrlChangeAsync( public static Task ClickOnWithScriptAsync(this UITestContext context, By by) => context.Get(by).ClickWithScriptAsync(context); + /// + /// Clicks through a path of admin menu items, ensuring that the next steps is visible before trying to click. + /// + public static async Task ClickThroughAdminMenuAsync(this UITestContext context, params By[] selectors) + { + var menuAnimationTime = TimeSpan.FromSeconds(1); + + for (var i = 0; i < selectors.Length - 1; i++) + { + // Only click on this menu item if the next item is not visible because this menu is already open. + if (context.Exists(selectors[i + 1].Safely().Within(menuAnimationTime))) continue; + + // The menus have animations, which interfere with the click being recognized. + await context.ClickReliablyOnAsync(selectors[i]); + + context.Exists(selectors[i + 1]); + } + + // The last item is clicked separately, because we don't do look-ahead checks here. + await context.ClickReliablyOnAsync(selectors[^1]); + } + + /// + /// Clicks through a path of admin menu items, ensuring that the next steps is visible before trying to click. + /// + public static Task ClickThroughAdminMenuAsync(this UITestContext context, params string[] ids) => + context.ClickThroughAdminMenuAsync(ids.Select(By.Id).ToArray()); + /// /// Switches control to JS alert box, accepts it, and switches control back to main document or first frame. /// diff --git a/Lombiq.Tests.UI/Extensions/OrchardCoreConfigurationExtensions.cs b/Lombiq.Tests.UI/Extensions/OrchardCoreConfigurationExtensions.cs index 50042180b..6c45c7458 100644 --- a/Lombiq.Tests.UI/Extensions/OrchardCoreConfigurationExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/OrchardCoreConfigurationExtensions.cs @@ -1,5 +1,5 @@ using Lombiq.Tests.UI.Services; -using OrchardCore.Search.Elasticsearch.Core.Recipes; +using OrchardCore.Elasticsearch.Core.Recipes; using System; using System.Diagnostics.CodeAnalysis; using System.Threading.Tasks; @@ -16,7 +16,7 @@ public static void ConfigureElasticSearchPrefix(this OrchardCoreConfiguration co /// /// Configure the app settings to use the provided in the Elasticsearch indexes created by - /// the . + /// the . /// public static void ConfigureElasticsearchPrefix(this OrchardCoreConfiguration configuration, string prefix) => configuration.BeforeAppStart += (_, arguments) => diff --git a/Lombiq.Tests.UI/Extensions/SeleniumResponseCompletedEventExtensions.cs b/Lombiq.Tests.UI/Extensions/SeleniumResponseCompletedEventExtensions.cs index 71840d49a..1f384a7ce 100644 --- a/Lombiq.Tests.UI/Extensions/SeleniumResponseCompletedEventExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/SeleniumResponseCompletedEventExtensions.cs @@ -23,20 +23,15 @@ public static string ToFormattedString(this ResponseData response) => $"Body size: {response.BodySize}{Environment.NewLine}" + $"Response content: {response.Content} {Environment.NewLine}"); - public static bool IsNonSuccessResponse(this ResponseCompletedEventArgs eventArgs) => - OrchardCoreUITestExecutorConfiguration.IsNonSuccessResponse(eventArgs); - - public static bool IsNonSuccessResponseAndNotExpectedNotFoundResponse(this ResponseCompletedEventArgs eventArgs, string urlContains) => - IsNonSuccessResponse(eventArgs) && - !eventArgs.IsNotFoundResponse(urlContains); - - public static bool IsNonSuccessResponseAndNotExpectedStatusResponse(this ResponseCompletedEventArgs eventArgs, string urlContains, int status) => - IsNonSuccessResponse(eventArgs) && - !(eventArgs.Response.Url.ContainsOrdinalIgnoreCase(urlContains) && eventArgs.Response.Status == status); - - public static bool IsNotFoundResponse(this ResponseCompletedEventArgs eventArgs, string urlContains) => - IsNotFoundResponse(eventArgs.Response, urlContains); - - public static bool IsNotFoundResponse(this ResponseData response, string urlContains) => - response.Status == 404 && response.Url.ContainsOrdinalIgnoreCase(urlContains); + public static void WithIgnoreExpectedNotFoundResponseFilter( + this OrchardCoreUITestExecutorConfiguration configuration, + string urlContains) => + configuration.WithIgnoreExpectedStatusResponseFilter(urlContains, 404); + + public static void WithIgnoreExpectedStatusResponseFilter( + this OrchardCoreUITestExecutorConfiguration configuration, + string urlContains, + int status) => + configuration.ResponseLogFilters[$"Ignore expected {status.ToTechnicalString()} error at {urlContains}."] = + eventArgs => !(eventArgs.Response.Url.ContainsOrdinalIgnoreCase(urlContains) && eventArgs.Response.Status == status); } diff --git a/Lombiq.Tests.UI/Extensions/ShortcutsUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/ShortcutsUITestContextExtensions.cs index 2225a4348..efdd07f7c 100644 --- a/Lombiq.Tests.UI/Extensions/ShortcutsUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/ShortcutsUITestContextExtensions.cs @@ -268,6 +268,12 @@ public static Task EnableFeatureDirectlyAsync( tenant, activateShell); + /// + /// Enables the Tenants feature for the default tenant. + /// + public static Task EnableTenantsFeatureDirectlyAsync(this UITestContext context) => + context.EnableFeatureDirectlyAsync("OrchardCore.Tenants", ShellSettings.DefaultShellName); + /// /// Disables the feature with the given directly. /// @@ -290,6 +296,26 @@ public static Task DisableFeatureDirectlyAsync( tenant, activateShell); + /// + /// Enables the theme with the given directly. + /// + public static async Task EnableThemeDirectlyAsync( + this UITestContext context, + string themeId, + bool isAdmin = false, + string tenant = null, + bool activateShell = true) + { + await context.EnableFeatureDirectlyAsync(themeId, tenant, activateShell); + await UsingScopeAsync( + context, + serviceProvider => isAdmin + ? serviceProvider.GetRequiredService().SetAdminThemeAsync(themeId) + : serviceProvider.GetRequiredService().SetSiteThemeAsync(themeId), + tenant, + activateShell); + } + /// /// Turns the Lombiq.Tests.UI.Shortcuts.FeatureToggleTestBench feature on, then off, and checks if the /// operations indeed worked. This can be used to test if anything breaks when a feature is enabled or disabled. @@ -493,14 +519,20 @@ public static async Task CreateAndSwitchToTenantAsync( string name, string urlPrefix, OrchardCoreSetupParameters setupParameters, - string featureProfile = null) + string featureProfile = null, + bool enableTenantsFeature = true) { + if (enableTenantsFeature) + { + await context.EnableTenantsFeatureDirectlyAsync(); + } + setupParameters ??= new OrchardCoreSetupParameters(context); var databaseProvider = setupParameters.DatabaseProvider == OrchardCoreSetupParameters.DatabaseType.SqlConnection ? DatabaseProviderValue.SqlConnection : setupParameters.DatabaseProvider.ToString(); - await context.Application.UsingScopeAsync( + await context.Application.UsingScopeServiceProviderAsync( async serviceProvider => { var shellHost = serviceProvider.GetRequiredService(); @@ -523,7 +555,7 @@ await context.Application.UsingScopeAsync( await shellHost.UpdateShellSettingsAsync(shellSettings); }); - await context.Application.UsingScopeAsync( + await context.Application.UsingScopeServiceProviderAsync( async serviceProvider => { var setupService = serviceProvider.GetRequiredService(); @@ -702,7 +734,7 @@ private static Task UsingScopeAsync( tenant ??= context.TenantName; if (tenant.StartsWith('!')) tenant = ShellSettings.DefaultShellName; - return context.Application.UsingScopeAsync(execute, tenant, activateShell); + return context.Application.UsingScopeServiceProviderAsync(execute, tenant, activateShell); } /// diff --git a/Lombiq.Tests.UI/Extensions/TenantsUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/TenantsUITestContextExtensions.cs index f769b9a41..81c0bc480 100644 --- a/Lombiq.Tests.UI/Extensions/TenantsUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/TenantsUITestContextExtensions.cs @@ -28,8 +28,15 @@ public static async Task CreateTenantManuallyAsync( string urlPrefix = "", string urlHost = "", string featureProfile = "", - bool navigate = true) + bool navigate = true, + bool enableTenantsFeature = true) { + if (enableTenantsFeature) + { + await context.EnableTenantsFeatureDirectlyAsync(); + await context.EnableFeatureDirectlyAsync("OrchardCore.Tenants.FeatureProfiles"); + } + if (navigate) { await context.GoToAdminRelativeUrlAsync("/Tenants"); diff --git a/Lombiq.Tests.UI/Extensions/TestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/TestContextExtensions.cs index 4bf7b4220..638569f5a 100644 --- a/Lombiq.Tests.UI/Extensions/TestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/TestContextExtensions.cs @@ -7,16 +7,12 @@ public static class TestContextExtensions { [Obsolete("Use GetElasticsearchSafeIndexName instead. This method will be removed in a future version.")] public static string GetElasticserachSafeIndexName(this ITestContext context, Guid id) => - context.GetElasticsearchSafeIndexName(id); + context.GetElasticsearchSafeIndexName(); /// - /// Gets a which is safe to use as an Elasticsearch index. + /// Generates a safe Elasticsearch index name from the test identifiers in . /// - /// - /// A unique identifier that stays the same between setup and test. This ensures that leftover data in the test - /// won't be confused with previous runs. - /// - public static string GetElasticsearchSafeIndexName(this ITestContext context, Guid id) + public static string GetElasticsearchSafeIndexName(this ITestContext context) { // Elasticsearch indexes are lowercase only. #pragma warning disable CA1308 // Normalize strings to uppercase @@ -28,12 +24,14 @@ public static string GetElasticsearchSafeIndexName(this ITestContext context, Gu .Trim('-'); #pragma warning restore CA1308 // Normalize strings to uppercase - if (string.IsNullOrWhiteSpace(name)) return id.ToString("N"); + var id = context?.Test?.UniqueID?.NullIfWhiteSpace() ?? Guid.NewGuid().ToString("N"); + + if (string.IsNullOrWhiteSpace(name)) return id; // An Elasticsearch index can't be longer than 255 character, but that includes the test name, tenant name, GUID // and relative index name. So altogether 100 characters is a reasonable limit for the test name prefix. if (name.Length > 100) name = name[..100]; - return $"{name}-{id:N}"; + return $"{name}-{id}"; } } diff --git a/Lombiq.Tests.UI/Extensions/UsingScopeWebApplicationInstanceExtensions.cs b/Lombiq.Tests.UI/Extensions/UsingScopeWebApplicationInstanceExtensions.cs index 00cbad113..a2f9d7eb2 100644 --- a/Lombiq.Tests.UI/Extensions/UsingScopeWebApplicationInstanceExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/UsingScopeWebApplicationInstanceExtensions.cs @@ -15,7 +15,7 @@ public static class UsingScopeWebApplicationInstanceExtensions /// Executes a delegate using the shell scope given by in an isolated async flow, while /// managing the shell state and invoking tenant events. /// - public static Task UsingScopeAsync( + public static Task UsingScopeServiceProviderAsync( this IWebApplicationInstance instance, Func execute, string tenant = "Default", diff --git a/Lombiq.Tests.UI/Extensions/VisualVerificationUITestContextExtensions.cs b/Lombiq.Tests.UI/Extensions/VisualVerificationUITestContextExtensions.cs index 65d8d0144..935c91349 100644 --- a/Lombiq.Tests.UI/Extensions/VisualVerificationUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/Extensions/VisualVerificationUITestContextExtensions.cs @@ -19,6 +19,7 @@ using System.Reflection; using System.Runtime.CompilerServices; using System.Text; +using System.Threading.Tasks; namespace Lombiq.Tests.UI.Extensions; @@ -48,19 +49,23 @@ public static class VisualVerificationUITestContextExtensions /// /// [VisualVerificationApprovedMethod] - public static void AssertVisualVerificationOnAllResolutions( + public static async Task AssertVisualVerificationOnAllResolutionsAsync( this UITestContext context, IEnumerable sizes, Func getSelector, double pixelErrorPercentageThreshold = 0, - Action configurator = null) + Action configurator = null, + TimeSpan? resolutionChangeDelay = null) { - context.HideScrollbar(); - var exceptions = new List(); foreach (var size in sizes) { + // Waiting after viewport size change ensures that any animation (such as sticky mobile menu) has time to + // transition to its final state. context.SetViewportSize(size); + await Task.Delay( + resolutionChangeDelay ?? TimeSpan.FromSeconds(1), + context.Configuration.TestCancellationToken); try { @@ -115,17 +120,19 @@ public static void AssertVisualVerificationOnAllResolutions( /// /// [VisualVerificationApprovedMethod] - public static void AssertVisualVerificationApprovedOnAllResolutionsWithPlatformSuffix( + public static Task AssertVisualVerificationApprovedOnAllResolutionsWithPlatformSuffixAsync( this UITestContext context, IEnumerable sizes, Func getSelector, double pixelErrorPercentageThreshold = 0, - Action configurator = null) => - context.AssertVisualVerificationOnAllResolutions( + Action configurator = null, + TimeSpan? resolutionChangeDelay = null) => + context.AssertVisualVerificationOnAllResolutionsAsync( sizes, getSelector, pixelErrorPercentageThreshold, - configuration => configuration.WithUsePlatformAsSuffix()); + configuration => configuration.WithUsePlatformAsSuffix(), + resolutionChangeDelay); /// /// Compares the baseline image and screenshot of the whole page. @@ -342,7 +349,10 @@ private static void AssertVisualVerificationApproved( By elementSelector, Action comparator, Rectangle? regionOfInterest = null, - Action configurator = null) => + Action configurator = null) + { + context.ScrollTo(elementSelector); + context.HideScrollbar(); context.AssertVisualVerificationApproved( elementSelector is null ? null : context.Get(elementSelector), comparator, @@ -359,6 +369,8 @@ private static void AssertVisualVerificationApproved( .JoinNotNullOrEmpty("-") ); }); + context.RestoreHiddenScrollbar(); + } [VisualVerificationApprovedMethod] private static void AssertVisualVerificationApproved( @@ -388,6 +400,8 @@ private static void AssertVisualVerificationApproved( .FirstOrDefault(); } + testFrame ??= context.TestManifest?.StackFrame; + if (testFrame == null) { throw new VisualVerificationCallerMethodNotFoundException(); diff --git a/Lombiq.Tests.UI/Helpers/AppLogAssertionHelper.cs b/Lombiq.Tests.UI/Helpers/AppLogAssertionHelper.cs index 5ffc5b682..5986496bc 100644 --- a/Lombiq.Tests.UI/Helpers/AppLogAssertionHelper.cs +++ b/Lombiq.Tests.UI/Helpers/AppLogAssertionHelper.cs @@ -9,6 +9,7 @@ public static class AppLogAssertionHelper /// /// An wrapping . /// + [Obsolete("This is no longer necessary after https://github.com/OrchardCMS/OrchardCore/pull/18341.")] public static readonly Expression> NotMediaCacheEntriesPredicate = logEntry => NotMediaCacheEntries(logEntry); @@ -17,6 +18,7 @@ public static class AppLogAssertionHelper /// DefaultMediaFileStoreCacheFileProvider. These errors frequently happen during UI testing when using Azure /// Blob Storage for media storage. They're harmless, though. /// + [Obsolete("This is no longer necessary after https://github.com/OrchardCMS/OrchardCore/pull/18341.")] public static bool NotMediaCacheEntries(IApplicationLogEntry logEntry) => logEntry.Category != "OrchardCore.Media.Core.DefaultMediaFileStoreCacheFileProvider" || !logEntry.Message.StartsWithOrdinalIgnoreCase("Error deleting cache folder"); diff --git a/Lombiq.Tests.UI/Helpers/TeamsHelper.cs b/Lombiq.Tests.UI/Helpers/TeamsHelper.cs index 211b33932..6ed90584c 100644 --- a/Lombiq.Tests.UI/Helpers/TeamsHelper.cs +++ b/Lombiq.Tests.UI/Helpers/TeamsHelper.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Net.Http; +using System.Net.Mime; using System.Text; using System.Text.Json; using System.Threading.Tasks; @@ -93,7 +94,7 @@ public static async Task SendTeamsMessageAsync(string webho using var client = new HttpClient(); var adaptiveCardJson = JsonSerializer.Serialize(adaptiveCardContent); - using var data = new StringContent(adaptiveCardJson, Encoding.UTF8, "application/json"); + using var data = new StringContent(adaptiveCardJson, Encoding.UTF8, MediaTypeNames.Application.Json); var response = await client.PostAsync(webhookUrl, data, TestContext.Current.CancellationToken); diff --git a/Lombiq.Tests.UI/Helpers/WebAppConfigHelper.cs b/Lombiq.Tests.UI/Helpers/WebAppConfigHelper.cs index b48bb5601..643969eb3 100644 --- a/Lombiq.Tests.UI/Helpers/WebAppConfigHelper.cs +++ b/Lombiq.Tests.UI/Helpers/WebAppConfigHelper.cs @@ -12,10 +12,10 @@ public static class WebAppConfigHelper /// /// The web app's project name. /// - /// The name of the folder that corresponds to the .NET version in the build output folder (e.g. "net8.0"). + /// The name of the folder that corresponds to the .NET version in the build output folder (e.g. "net10.0"). /// /// The absolute path to the assembly (DLL) of the application being tested. - public static string GetAbsoluteApplicationAssemblyPath(string webAppName, string frameworkFolderName = "net8.0") + public static string GetAbsoluteApplicationAssemblyPath(string webAppName, string frameworkFolderName = "net10.0") { string baseDirectory; diff --git a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj index d3b588ed6..986dbf1e2 100644 --- a/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj +++ b/Lombiq.Tests.UI/Lombiq.Tests.UI.csproj @@ -1,9 +1,7 @@ - - + - net8.0 + net10.0 false - $(DefaultItemExcludes);.git* true @@ -12,85 +10,56 @@ true + + + Lombiq UI Testing Toolbox for Orchard Core - Lombiq Technologies - Copyright © 2020, Lombiq Technologies Ltd. - Lombiq UI Testing Toolbox for Orchard Core: Web UI testing toolbox mostly for Orchard Core applications. Everything you need to do UI testing with Selenium for an Orchard app is here. See the project website for detailed documentation. + 2020 + Web UI testing toolbox mostly for Orchard Core applications. Everything you need to do UI testing with Selenium for an Orchard app is here. OrchardCore;Lombiq;AspNetCore;Selenium;Atata;Shouldly;xUnit;Axe;AccessibilityTesting;UITesting;Testing;Automation;ZAP;Zed Attack Proxy;Security;Scanning;OWASP - NuGetIcon.png https://github.com/Lombiq/UI-Testing-Toolbox - https://github.com/Lombiq/UI-Testing-Toolbox - BSD-3-Clause - - - - PreserveNewest - - - - - - PreserveNewest - true - - - PreserveNewest - true - - - PreserveNewest - true - - - PreserveNewest - true - - - PreserveNewest - true - - - PreserveNewest - true - - - PreserveNewest - true - + + + + + + + + - + - + - + + - - + + - - - - + + + + - + - @@ -101,19 +70,24 @@ - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + @@ -140,4 +114,6 @@ + + diff --git a/Lombiq.Tests.UI/Models/AccessibilityCheckingResult.cs b/Lombiq.Tests.UI/Models/AccessibilityCheckingResult.cs new file mode 100644 index 000000000..4c4a82f06 --- /dev/null +++ b/Lombiq.Tests.UI/Models/AccessibilityCheckingResult.cs @@ -0,0 +1,32 @@ +using Deque.AxeCore.Commons; +using System; +using System.Collections.Generic; + +namespace Lombiq.Tests.UI.Models; + +public class AccessibilityCheckingResult +{ + public IList Violations { get; init; } = []; + public IList Passes { get; init; } = []; + public IList Inapplicable { get; init; } = []; + public IList Incomplete { get; init; } = []; + public DateTimeOffset? Timestamp { get; set; } + public AxeTestEnvironment TestEnvironment { get; set; } + public AxeTestRunner TestRunner { get; set; } + public string Url { get; set; } + public AxeTestEngine TestEngine { get; set; } + + public static implicit operator AccessibilityCheckingResult(AxeResult axeResult) => + new() + { + Violations = [.. axeResult.Violations], + Passes = [.. axeResult.Passes], + Inapplicable = [.. axeResult.Inapplicable], + Incomplete = [.. axeResult.Incomplete], + Timestamp = axeResult.Timestamp, + TestEnvironment = axeResult.TestEnvironment, + TestRunner = axeResult.TestRunner, + Url = axeResult.Url, + TestEngine = axeResult.TestEngine, + }; +} diff --git a/Lombiq.Tests.UI/Models/ElasticsearchRunningContext.cs b/Lombiq.Tests.UI/Models/ElasticsearchRunningContext.cs index 641ecc6db..ad8c1ae04 100644 --- a/Lombiq.Tests.UI/Models/ElasticsearchRunningContext.cs +++ b/Lombiq.Tests.UI/Models/ElasticsearchRunningContext.cs @@ -1,23 +1,29 @@ -using Elasticsearch.Net; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Core; using Lombiq.Tests.UI.Extensions; using Lombiq.Tests.UI.Services; using Microsoft.Extensions.DependencyInjection; -using Nest; +using OrchardCore.Elasticsearch.Core.Deployment; +using OrchardCore.Elasticsearch.Core.Models; +using OrchardCore.Elasticsearch.Core.Recipes; +using OrchardCore.Elasticsearch.Core.Services; using OrchardCore.Indexing; -using OrchardCore.Search.Elasticsearch.Core.Services; +using OrchardCore.Indexing.Core; +using OrchardCore.Recipes.Models; using System; using System.Diagnostics.CodeAnalysis; -using System.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; using System.Threading; using System.Threading.Tasks; namespace Lombiq.Tests.UI.Models; -public record ElasticsearchRunningContext(Guid Id, string Prefix) +public record ElasticsearchRunningContext(string Prefix) { /// /// Gets the expression that refers to all indexes that start with . This should only be used - /// with , because the OrchardCore-specific services automatically apply the prefix from + /// with , because the OrchardCore-specific services automatically apply the prefix from /// configuration so it would result in double prefixing. /// private IndexName LowLevelIndexName => Indices.Index($"{Prefix}_*"); @@ -25,121 +31,32 @@ public record ElasticsearchRunningContext(Guid Id, string Prefix) // Elasticsearch indexing sometimes takes longer, and the testing starts before indexing finishes. To prevent that, // we are checking if all indexing tasks are finished. public Task BeforeTestAsync(UITestContext context) => - context.Application.UsingScopeAsync(async provider => + context.Application.UsingScopeServiceProviderAsync(async provider => { - var index = LowLevelIndexName; - var testCancellationToken = context.Configuration.TestCancellationToken; - - if (GetClient(provider) is not { } client) + if (provider.GetService() is { } indexProfileManager) { - throw new InvalidOperationException( - $"Couldn't resolve {nameof(IElasticClient)} while waiting for \"{index}\"."); + await RebuildAllIndexesAsync(indexProfileManager, provider); } - (await client.Indices.FlushAsync(index, ct: testCancellationToken)).ThrowIfFailed($"flush index \"{index}\""); - (await client.Indices.RefreshAsync(index, ct: testCancellationToken)).ThrowIfFailed($"refresh index \"{index}\""); - - var settingsService = provider.GetRequiredService(); - var indexSettings = await settingsService.GetSettingsAsync(); - var exactIndexName = indexSettings.FirstOrDefault()?.IndexName; - - if (exactIndexName == null) return; - - var timeout = TimeSpan.FromSeconds(60); - - using var timeoutCancellationTokenSource = new CancellationTokenSource(timeout); - using var jointCancellationTokenSource = CancellationTokenSource - .CreateLinkedTokenSource(testCancellationToken, timeoutCancellationTokenSource.Token); - var jointCancellationToken = jointCancellationTokenSource.Token; - - var elasticIndexManager = provider.GetRequiredService(); - - long? lastFinishedTaskId = null; - - try - { - var lastTaskId = await GetLastTaskIdAsync(provider.GetRequiredService()); - - // We have the ID of the last indexing task that should happen, so we are waiting here for that task to - // complete, since "GetLastTaskId()" returns only completed tasks. - while (lastFinishedTaskId < lastTaskId || lastFinishedTaskId == null) - { - jointCancellationToken.ThrowIfCancellationRequested(); - - lastFinishedTaskId = await TryGetLastFinishedTaskIdAsync(elasticIndexManager, exactIndexName); - - // The indexing takes a couple of seconds, so there is no need to check them so fast: we are adding - // a delay. - await Task.Delay(500, jointCancellationToken); - } - } - catch (TaskCanceledException ex) + if (provider.GetService() is { } indexingService) { - throw new TaskCanceledException( - "Elasticsearch indexing wasn't finished due to " + - $"{(testCancellationToken.IsCancellationRequested ? "the test being canceled" : $"it not completing within {timeout}")}.", - ex, - jointCancellationToken); + await indexingService.ProcessRecordsForAllIndexesAsync(); } }); - public Task AfterTestAsync(UITestContext context) => - context?.Application?.Services is { } ? AfterTestInnerAsync(context) : Task.CompletedTask; - - private static async Task GetLastTaskIdAsync(IIndexingTaskManager indexingTaskManager) + public async Task AfterTestAsync(UITestContext context) { - const int batchSize = 1000; - long lastTaskId = 0; - bool hasTask = true; - - // We are getting the last indexing task (regardless of the state). This function works like a cursor, so - // there is no way to get the last task in the list directly. Since we have to provide a "count" parameter, - // we are retrieving the indexing tasks by batches of 1000. Then if there are no more, we get the last one. - // We want to set "hasTask" inside the loop. -#pragma warning disable S1994 // "for" loop increment clauses should modify the loops' counters - for (var startIndex = 0; hasTask; startIndex += batchSize) + try { - var lastTask = (await indexingTaskManager.GetIndexingTasksAsync(startIndex, batchSize)) - .LastOrDefault(); - - hasTask = lastTask != null; - - if (hasTask) + if (context?.Application?.Services != null) { - lastTaskId = lastTask.Id; + await context.Application.UsingScopeServiceProviderAsync(provider => + WithPrefixElasticsearchIndexCleanupFinallyAsync(provider, context, LowLevelIndexName)); } } -#pragma warning restore S1994 // "for" loop increment clauses should modify the loops' counters - - return lastTaskId; - } - - /// - /// Asking for the last task ID can throw an exception if the underlying value is not initialized yet. This method - /// catches the exception and returns null instead so it can be safely retried. - /// - private static async Task TryGetLastFinishedTaskIdAsync(ElasticIndexManager elasticIndexManager, string indexName) - { - try - { - return await elasticIndexManager.GetLastTaskId(indexName); - } - catch (InvalidOperationException) - { - return null; - } - } - - private async Task AfterTestInnerAsync(UITestContext context) - { - try - { - await context.Application.UsingScopeAsync(provider => - WithPrefixElasticsearchIndexCleanupFinallyAsync(provider, context, LowLevelIndexName)); - } catch (Exception inner) { - context.Scope?.AtataContext?.Log?.Error(inner.ToString()); + context?.Scope?.AtataContext?.Log?.Error(inner.ToString()); } } @@ -147,23 +64,19 @@ await context.Application.UsingScopeAsync(provider => "Usage", "MA0040:Forward the CancellationToken parameter to methods that take one", Justification = "Cleanup code has no viable cancellation token because even failed tests should be cleaned up.")] - private static async Task WithPrefixElasticsearchIndexCleanupFinallyAsync( + private async Task WithPrefixElasticsearchIndexCleanupFinallyAsync( IServiceProvider provider, UITestContext context, IndexName index) { - static async Task CheckIfIndexExistsAsync(IElasticClient client, IndexName index) + static async Task CheckIfIndexExistsAsync(ElasticsearchClient client, IndexName index) { - var indices = (await client.Indices.GetAsync(index, ct: CancellationToken.None)) + var indices = (await client.Indices.GetAsync(index, cancellationToken: CancellationToken.None)) .ThrowIfFailed($"query index \"{index}\""); return indices.Indices.Count > 0; } - if (GetClient(provider) is not { } client) - { - throw new InvalidOperationException( - $"Couldn't resolve {nameof(IElasticClient)} while attempting to clean up \"{index}\"."); - } + var client = GetClient(provider); if (!await CheckIfIndexExistsAsync(client, index)) { @@ -171,15 +84,43 @@ static async Task CheckIfIndexExistsAsync(IElasticClient client, IndexName return; } - var deleteRequest = new DeleteIndexRequest(index) { ExpandWildcards = ExpandWildcards.All }; - (await client.Indices.DeleteAsync(deleteRequest)).ThrowIfFailed($"delete index \"{index}\""); - + await client.DeleteAllIndexesAsync(Prefix); if (await CheckIfIndexExistsAsync(client, index)) { - throw new InvalidOperationException($"Couldn't delete indexes for \"{index.Name}\"."); + throw new InvalidOperationException($"Couldn't delete indexes for \"{index}\"."); } } - private static IElasticClient GetClient(IServiceProvider provider) => - provider.GetService() ?? provider.GetService(); + private static ElasticsearchClient GetClient(IServiceProvider provider) + { + if (provider.GetService() is { } existingClient) + { + return existingClient; + } + + if (provider.GetService() is { } factory && + factory.Create(new ElasticsearchConnectionOptions()) is { } factoryClient) + { + return factoryClient; + } + + throw new InvalidOperationException( + $"Couldn't resolve {nameof(ElasticsearchClient)}."); + } + + private static Task RebuildAllIndexesAsync( + IIndexProfileManager indexProfileManager, + IServiceProvider serviceProvider) + { + var step = new ElasticsearchIndexRebuildStep(indexProfileManager, serviceProvider); + var model = new ElasticsearchIndexRebuildDeploymentStep { IncludeAll = true }; + var context = new RecipeExecutionContext + { + ExecutionId = Guid.NewGuid().ToString(), + Name = "elastic-index-rebuild", + Step = (JsonObject)JsonSerializer.SerializeToNode(model), + }; + + return step.ExecuteAsync(context); + } } diff --git a/Lombiq.Tests.UI/Models/JsonHtmlValidationError.cs b/Lombiq.Tests.UI/Models/HtmlValidationError.cs similarity index 95% rename from Lombiq.Tests.UI/Models/JsonHtmlValidationError.cs rename to Lombiq.Tests.UI/Models/HtmlValidationError.cs index f5892f8c6..618d4d988 100644 --- a/Lombiq.Tests.UI/Models/JsonHtmlValidationError.cs +++ b/Lombiq.Tests.UI/Models/HtmlValidationError.cs @@ -3,7 +3,7 @@ namespace Lombiq.Tests.UI.Models; -public class JsonHtmlValidationError +public class HtmlValidationError { [JsonPropertyName("ruleId")] public string RuleId { get; set; } diff --git a/Lombiq.Tests.UI/Models/UITestManifest.cs b/Lombiq.Tests.UI/Models/UITestManifest.cs index 347c8779a..d5a7fd1a7 100644 --- a/Lombiq.Tests.UI/Models/UITestManifest.cs +++ b/Lombiq.Tests.UI/Models/UITestManifest.cs @@ -1,5 +1,9 @@ +#nullable enable + using Lombiq.Tests.UI.Services; using System; +using System.Diagnostics; +using System.Linq; using System.Threading.Tasks; using Xunit; using Xunit.Sdk; @@ -11,9 +15,18 @@ namespace Lombiq.Tests.UI.Models; /// public class UITestManifest { - public ITest XunitTest => TestContext.Current.Test; - public string Name => XunitTest.TestDisplayName; - public Func TestAsync { get; private set; } + public ITest? XunitTest => TestContext.Current.Test; + public string? Name => XunitTest?.TestDisplayName; + public Func TestAsync { get; } + public EnhancedStackFrame? StackFrame { get; } + + public UITestManifest(Func testAsync) + { + TestAsync = testAsync; - public UITestManifest(Func testAsync) => TestAsync = testAsync; + var typeName = testAsync.Method.DeclaringType?.FullName?.Split('+')[0]; + var methodName = testAsync.Method.Name.StartsWith('<') ? testAsync.Method.Name[1..].Split('>')[0] : testAsync.Method.Name; + StackFrame = new EnhancedStackTrace(new StackTrace(fNeedFileInfo: true)) + .FirstOrDefault(frame => frame.MethodInfo.DeclaringType?.FullName == typeName && frame.MethodInfo.Name == methodName); + } } diff --git a/Lombiq.Tests.UI/Models/VisualVerificationMatchApprovedContext.cs b/Lombiq.Tests.UI/Models/VisualVerificationMatchApprovedContext.cs index e08eb16bb..6be61ce46 100644 --- a/Lombiq.Tests.UI/Models/VisualVerificationMatchApprovedContext.cs +++ b/Lombiq.Tests.UI/Models/VisualVerificationMatchApprovedContext.cs @@ -55,7 +55,7 @@ private static string GetModuleName(EnhancedStackFrame frame) } while (currentMethod is not null); - var depthMark = DepthMark(); + var depthMark = DepthMark; if (depthMark.IsMatch(moduleName)) { moduleName = depthMark.Match(moduleName).Groups["module"].Value; @@ -69,7 +69,7 @@ private static string GetModuleName(EnhancedStackFrame frame) private static string GetMethodName(EnhancedStackFrame frame) { var methodName = frame.MethodInfo.Name!; - var inheritedMethod = InheritedMethod(); + var inheritedMethod = InheritedMethod; if (inheritedMethod.IsMatch(methodName)) { methodName = inheritedMethod.Match(methodName).Groups["method"].Value; @@ -79,8 +79,8 @@ private static string GetMethodName(EnhancedStackFrame frame) } [GeneratedRegex("^(?.*)`[0-9]+$", RegexOptions.ExplicitCapture)] - private static partial Regex DepthMark(); + private static partial Regex DepthMark { get; } [GeneratedRegex("^<(?.*)>.*$", RegexOptions.ExplicitCapture)] - private static partial Regex InheritedMethod(); + private static partial Regex InheritedMethod { get; } } diff --git a/Lombiq.Tests.UI/OrchardCoreUITestBase.cs b/Lombiq.Tests.UI/OrchardCoreUITestBase.cs index f8e394ef4..d90449d83 100644 --- a/Lombiq.Tests.UI/OrchardCoreUITestBase.cs +++ b/Lombiq.Tests.UI/OrchardCoreUITestBase.cs @@ -404,8 +404,7 @@ protected virtual async Task ExecuteTestAsync( await changeConfigurationAsync.InvokeFuncAsync(configuration); await ExecuteOrchardCoreTestAsync( - (configuration, contextId) => - new OrchardCoreInstance(configuration.OrchardCoreConfiguration, contextId, configuration.TestOutputHelper), + (configuration, contextId) => new OrchardCoreInstance(configuration, contextId), testManifest, configuration); } diff --git a/Lombiq.Tests.UI/SecurityScanning/OrchardCoreUITestExecutorConfigurationExtensions.cs b/Lombiq.Tests.UI/SecurityScanning/OrchardCoreUITestExecutorConfigurationExtensions.cs index 34b5dbf61..0fe4f506d 100644 --- a/Lombiq.Tests.UI/SecurityScanning/OrchardCoreUITestExecutorConfigurationExtensions.cs +++ b/Lombiq.Tests.UI/SecurityScanning/OrchardCoreUITestExecutorConfigurationExtensions.cs @@ -1,5 +1,4 @@ using Lombiq.Tests.UI.Extensions; -using Lombiq.Tests.UI.Helpers; using Lombiq.Tests.UI.Services; using Lombiq.Tests.UI.Shortcuts.Controllers; using Microsoft.Extensions.Logging; @@ -73,7 +72,6 @@ public static Func CreateAppLogAssertionForSecuri app.LogsShouldNotContainAsync( logEntry => logEntry.Level >= LogLevel.Error && - AppLogAssertionHelper.NotMediaCacheEntries(logEntry) && !permittedErrorLinePatterns.Any(pattern => Regex.IsMatch(logEntry.ToString(), pattern, RegexOptions.IgnoreCase | RegexOptions.Compiled)), TestContext.Current.CancellationToken); diff --git a/Lombiq.Tests.UI/SecurityScanning/SecurityScanningUITestContextExtensions.cs b/Lombiq.Tests.UI/SecurityScanning/SecurityScanningUITestContextExtensions.cs index 947686bcf..1496d97d6 100644 --- a/Lombiq.Tests.UI/SecurityScanning/SecurityScanningUITestContextExtensions.cs +++ b/Lombiq.Tests.UI/SecurityScanning/SecurityScanningUITestContextExtensions.cs @@ -123,7 +123,7 @@ public static Task RunAndAssertGraphQLSecurityScanAsync( /// /// /// The of the JSON OpenAPI definition for the API to scan. If then the API - /// of the app will automatically be discovered with Swagger. + /// of the app will automatically be discovered with OpenAPI. /// /// A delegate to configure the security scan in detail. /// @@ -137,7 +137,7 @@ public static async Task RunAndAssertOpenApiSecurityScanAsync( { if (apiDefinitionUri == null) { - await context.EnableFeatureDirectlyAsync(Shortcuts.ShortcutsFeatureIds.Swagger); + await context.EnableFeatureDirectlyAsync(Shortcuts.ShortcutsFeatureIds.OpenApi); } await context.RunAndAssertSecurityScanAsync( @@ -152,7 +152,7 @@ await context.RunAndAssertSecurityScanAsync( "No job named \"openapi\" found in the Automation Framework Plan. We can only run the " + "OpenAPI scan if the job exists."); - apiDefinitionUri ??= context.GetAbsoluteUri("/swagger/v1/swagger.json"); + apiDefinitionUri ??= context.GetAbsoluteUri("/openapi/v1.json"); openApiJob.GetOrCreateParameters().SetMappingChild("apiUrl", apiDefinitionUri.ToString()); }); diff --git a/Lombiq.Tests.UI/Services/AccessibilityCheckingConfiguration.cs b/Lombiq.Tests.UI/Services/AccessibilityCheckingConfiguration.cs index 65de46e0f..5781334fe 100644 --- a/Lombiq.Tests.UI/Services/AccessibilityCheckingConfiguration.cs +++ b/Lombiq.Tests.UI/Services/AccessibilityCheckingConfiguration.cs @@ -2,6 +2,7 @@ using Deque.AxeCore.Selenium; using Lombiq.Tests.UI.Extensions; using Lombiq.Tests.UI.Helpers; +using Lombiq.Tests.UI.Models; using System; using System.Collections.Generic; using System.Linq; @@ -47,10 +48,24 @@ public class AccessibilityCheckingConfiguration EnableOnValidatablePagesAccessibilityCheckingAndAssertionOnPageChangeRule; /// - /// Gets or sets a delegate to run assertions on the when accessibility checking happens. - /// Defaults to . + /// Gets a collection of delegates that select which is + /// retained. If there are more than one filters, all of them must return . /// - public Action AssertAxeResult { get; set; } = AssertAxeResultIsEmpty; + public IDictionary> AxeResultIncompleteFilters { get; } = + new Dictionary>(); + + /// + /// Gets a collection of delegates that select which is + /// retained. If there are more than one filters, all of them must return . + /// + public IDictionary> AxeResultViolationsFilters { get; } = + new Dictionary>(); + + /// + /// Gets or sets a delegate to run assertions on the when accessibility + /// checking happens. Defaults to . + /// + public Action AssertAxeResult { get; set; } = AssertAxeResultIsEmpty; /// /// Configures the given to check for WCAG 2.1 AA compliance. Use the newer @@ -77,7 +92,7 @@ public class AccessibilityCheckingConfiguration public static readonly Func ConfigureWcag22aaWithBestPractices = axeBuilder => axeBuilder.WithTags("wcag2a", "wcag2aa", "wcag21a", "wcag21aa", "wcag22a", "wcag22aa", "best-practice"); - public static readonly Action AssertAxeResultIsEmpty = axeResult => + public static readonly Action AssertAxeResultIsEmpty = axeResult => { axeResult.Violations.AxeResultItemsShouldBeEmpty(); axeResult.Incomplete.AxeResultItemsShouldBeEmpty(); diff --git a/Lombiq.Tests.UI/Services/BrowserConfiguration.cs b/Lombiq.Tests.UI/Services/BrowserConfiguration.cs index d8a2c5ba3..b7a2ee199 100644 --- a/Lombiq.Tests.UI/Services/BrowserConfiguration.cs +++ b/Lombiq.Tests.UI/Services/BrowserConfiguration.cs @@ -55,4 +55,10 @@ public class BrowserConfiguration /// exist yet. /// internal string UITestContextId { get; set; } + + /// + /// Gets or sets the , to be used during + /// driver creation when the does not exist yet. + /// + internal string TempDirectoryPath { get; set; } } diff --git a/Lombiq.Tests.UI/Services/HtmlValidationConfiguration.cs b/Lombiq.Tests.UI/Services/HtmlValidationConfiguration.cs index 5aa04d86c..1e6543a68 100644 --- a/Lombiq.Tests.UI/Services/HtmlValidationConfiguration.cs +++ b/Lombiq.Tests.UI/Services/HtmlValidationConfiguration.cs @@ -1,9 +1,9 @@ using Atata.Cli.HtmlValidate; using Atata.HtmlValidation; -using Lombiq.Tests.UI.Extensions; using Lombiq.Tests.UI.Helpers; -using Shouldly; +using Lombiq.Tests.UI.Models; using System; +using System.Collections.Generic; using System.IO; using System.Threading.Tasks; @@ -52,11 +52,19 @@ public class HtmlValidationConfiguration /// public Action HtmlValidationOptionsAdjuster { get; set; } + /// + /// Gets a dictionary of filters. The errors from the are filtered by + /// each entry and only those are kept that pass each filter entry. + /// + public IDictionary> HtmlValidationFilters { get; } = + new Dictionary>(); + /// /// Gets or sets a delegate to run assertions on the when HTML validation - /// happens. Defaults to . + /// happens. If you only want to filter the validation errors, use or . /// - public Func AssertHtmlValidationResultAsync { get; set; } = AssertHtmlValidationOutputIsEmptyAsync; + public Func, Task> AssertHtmlValidationResultAsync { get; set; } /// /// Gets or sets a value indicating whether to automatically run HTML validation every time a page changes (either @@ -88,23 +96,36 @@ public HtmlValidationConfiguration WithRelativeConfigPath(params string[] pathSe return this; } - public static readonly Func AssertHtmlValidationOutputIsEmptyAsync = - validationResult => - { - // Keep supporting cases where output format is not set to JSON. - if (validationResult.Output.Trim().StartsWith('[') || - validationResult.Output.Trim().StartsWith('{')) - { - var errors = validationResult.GetParsedErrors(); - errors.ShouldBeEmpty(HtmlValidationResultExtensions.GetParsedErrorMessageString(errors)); - } - else - { - validationResult.Output.ShouldBeEmpty(); - } + /// + /// Updates the . + /// + public HtmlValidationConfiguration WithFilters(string name, Func filter) + { + HtmlValidationFilters[name] = filter; + + return this; + } + + /// + /// Updates the with the OC-15222 key to handle a specific bug. + /// + /// + /// Rule exclusions due to https://github.com/OrchardCMS/OrchardCore/issues/15222, usages can be removed once it is + /// resolved. + /// + public HtmlValidationConfiguration WithOC15222Filter() => + WithFilters("OC-15222", error => + error.RuleId is not ("prefer-native-element" or "text-content" or "no-redundant-role")); - return Task.CompletedTask; - }; + /// + /// Updates the with the OC-17907 key to handle a specific bug. + /// + /// + /// Rule exclusions due to https://github.com/OrchardCMS/OrchardCore/issues/17907, usages can be removed once it is + /// resolved. + /// + public HtmlValidationConfiguration WithOC17907Filter() => + WithFilters("OC-17907", error => error.RuleId is not "attribute-misuse"); public static readonly Predicate EnableOnValidatablePagesHtmlValidationAndAssertionOnPageChangeRule = UrlCheckHelper.IsValidatablePage; diff --git a/Lombiq.Tests.UI/Services/OrchardCoreHosting/OrchardApplicationFactory.cs b/Lombiq.Tests.UI/Services/OrchardCoreHosting/OrchardApplicationFactory.cs index 4bd0c39b4..c6141ddda 100644 --- a/Lombiq.Tests.UI/Services/OrchardCoreHosting/OrchardApplicationFactory.cs +++ b/Lombiq.Tests.UI/Services/OrchardCoreHosting/OrchardApplicationFactory.cs @@ -122,5 +122,5 @@ public override async ValueTask DisposeAsync() internal static class OrchardApplicationFactoryCounter { - public static object CreateHostLock { get; } = new(); + public static Lock CreateHostLock { get; } = new(); } diff --git a/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs b/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs index 121ff2fa9..783c7d1dc 100644 --- a/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs +++ b/Lombiq.Tests.UI/Services/OrchardCoreInstance.cs @@ -1,6 +1,5 @@ using Lombiq.HelpfulLibraries.Common.Utilities; using Lombiq.Tests.Integration.Services; -using Lombiq.Tests.UI.Constants; using Lombiq.Tests.UI.Helpers; using Lombiq.Tests.UI.Models; using Lombiq.Tests.UI.Services.OrchardCoreHosting; @@ -56,9 +55,8 @@ static OrchardCoreInstanceCounter() public sealed class OrchardCoreInstance : IWebApplicationInstance where TEntryPoint : class { - private readonly OrchardCoreConfiguration _configuration; + private readonly OrchardCoreUITestExecutorConfiguration _configuration; private readonly string _contextId; - private readonly ITestOutputHelper _testOutputHelper; private readonly CancellationTokenSource _cancellationTokenSource = new(); private string _contentRootPath; @@ -68,24 +66,25 @@ public sealed class OrchardCoreInstance : IWebApplicationInstance private TestReverseProxy _reverseProxy; public IServiceProvider Services => _orchardApplication?.Services; + public OrchardCoreConfiguration OrchardCoreConfiguration => _configuration.OrchardCoreConfiguration; + public ITestOutputHelper TestOutputHelper => _configuration.TestOutputHelper; - public OrchardCoreInstance(OrchardCoreConfiguration configuration, string contextId, ITestOutputHelper testOutputHelper) + public OrchardCoreInstance(OrchardCoreUITestExecutorConfiguration configuration, string contextId) { _configuration = configuration; _contextId = contextId; - _testOutputHelper = testOutputHelper; } public async Task StartUpAsync() { _url = await OrchardCoreInstanceCounter.GetNewUriAsync(); - _testOutputHelper.WriteLineTimestampedAndDebug("The generated URL for the Orchard Core instance is \"{0}\".", _url.AbsoluteUri); + TestOutputHelper.WriteLineTimestampedAndDebug("The generated URL for the Orchard Core instance is \"{0}\".", _url.AbsoluteUri); CreateContentRootFolder(); - if (!string.IsNullOrEmpty(_configuration.SnapshotDirectoryPath) && Directory.Exists(_configuration.SnapshotDirectoryPath)) + if (!string.IsNullOrEmpty(OrchardCoreConfiguration.SnapshotDirectoryPath) && Directory.Exists(OrchardCoreConfiguration.SnapshotDirectoryPath)) { - FileSystem.CopyDirectory(_configuration.SnapshotDirectoryPath, _contentRootPath, overwrite: true); + FileSystem.CopyDirectory(OrchardCoreConfiguration.SnapshotDirectoryPath, _contentRootPath, overwrite: true); } else { @@ -102,7 +101,7 @@ public async Task StartUpAsync() await _reverseProxy.StartAsync(); - _configuration.StartCount++; + OrchardCoreConfiguration.StartCount++; await StartOrchardAppAsync(); return _url; @@ -156,18 +155,18 @@ public async ValueTask DisposeAsync() private void CreateContentRootFolder() { - _contentRootPath = DirectoryPaths.GetTempDirectoryPath(_contextId, "App"); + _contentRootPath = _configuration.GetTempDirectoryPathWithFallback(_contextId, "App"); Directory.CreateDirectory(_contentRootPath); - _testOutputHelper.WriteLineTimestampedAndDebug("Content root path was created: {0}", _contentRootPath); + TestOutputHelper.WriteLineTimestampedAndDebug("Content root path was created: {0}", _contentRootPath); } private async Task StartOrchardAppAsync() { - _testOutputHelper.WriteLineTimestampedAndDebug("Attempting to start the Orchard Core instance."); + TestOutputHelper.WriteLineTimestampedAndDebug("Attempting to start the Orchard Core instance."); var arguments = new InstanceCommandLineArgumentsBuilder(); - await _configuration.BeforeAppStart + await OrchardCoreConfiguration.BeforeAppStart .InvokeAsync(handler => handler(CreateAppStartContext(), arguments)); // This is to avoid adding Razor runtime view compilation. @@ -187,9 +186,9 @@ await DirectoryHelper.SafelyDeleteDirectoryIfExistsAsync( options.FilteredLevels.Add(LogLevel.Error); options.FilteredLevels.Add(LogLevel.Critical); options.OutputFormatter = FakeLoggerApplicationLogEntry.FormatLogRecord; - options.OutputSink += message => _testOutputHelper.WriteLine(message); + options.OutputSink += message => TestOutputHelper.WriteLine(message); - _configuration.AfterFakeLoggingConfiguration?.Invoke(CreateAppStartContext(), options); + OrchardCoreConfiguration.AfterFakeLoggingConfiguration?.Invoke(CreateAppStartContext(), options); })), (configuration, orchardBuilder) => orchardBuilder .ConfigureUITesting(configuration, enableShortcutsDuringUITesting: true), @@ -199,7 +198,7 @@ await DirectoryHelper.SafelyDeleteDirectoryIfExistsAsync( _orchardApplication.ClientOptions.BaseAddress = new Uri(_reverseProxy.RootUrl); _reverseProxy.AttachConnectionProvider(_orchardApplication); - _testOutputHelper.WriteLineTimestampedAndDebug("The Orchard Core instance was started."); + TestOutputHelper.WriteLineTimestampedAndDebug("The Orchard Core instance was started."); } private async Task StopOrchardAppAsync() @@ -208,15 +207,15 @@ private async Task StopOrchardAppAsync() if (_orchardApplication == null) return; - _testOutputHelper.WriteLineTimestampedAndDebug("Attempting to stop the Orchard Core instance."); + TestOutputHelper.WriteLineTimestampedAndDebug("Attempting to stop the Orchard Core instance."); await _orchardApplication.DisposeAsync(); _orchardApplication = null; await OrchardCoreInstanceCounter.PortLeases.StopLeaseAsync(_url.Port, CancellationToken.None); - _testOutputHelper.WriteLineTimestampedAndDebug("The Orchard Core instance was stopped."); + TestOutputHelper.WriteLineTimestampedAndDebug("The Orchard Core instance was stopped."); - await _configuration.AfterAppStop + await OrchardCoreConfiguration.AfterAppStop .InvokeAsync(handler => handler(CreateAppStartContext())); } @@ -228,7 +227,7 @@ private async Task TakeSnapshotInnerAsync(string snapshotDirectoryPath) Directory.CreateDirectory(snapshotDirectoryPath); - await _configuration.BeforeTakeSnapshot + await OrchardCoreConfiguration.BeforeTakeSnapshot .InvokeAsync(handler => handler(CreateAppStartContext(), snapshotDirectoryPath)); FileSystem.CopyDirectory(_contentRootPath, snapshotDirectoryPath, overwrite: true); diff --git a/Lombiq.Tests.UI/Services/OrchardCoreSetupConfiguration.cs b/Lombiq.Tests.UI/Services/OrchardCoreSetupConfiguration.cs index 56aa6969e..ca281fb08 100644 --- a/Lombiq.Tests.UI/Services/OrchardCoreSetupConfiguration.cs +++ b/Lombiq.Tests.UI/Services/OrchardCoreSetupConfiguration.cs @@ -1,6 +1,7 @@ using Lombiq.Tests.UI.Constants; using System; using System.IO; +using System.Net.Http; using System.Threading.Tasks; namespace Lombiq.Tests.UI.Services; @@ -44,6 +45,14 @@ public class OrchardCoreSetupConfiguration /// public bool FastFailSetup { get; set; } = true; + /// + /// Gets or sets a value indicating whether an should be used to send a custom POST + /// message. If the value is , then Selenium browser automation is used to set the form + /// fields on the web browser GUI style. This is generally slower and should be only used to test basic Orchard Core + /// functionality or if the setup screen has been customized. + /// + public bool SetupWithHttpClient { get; set; } = true; + public string SetupSnapshotDirectoryPath { get; set; } = Path.Combine(DirectoryPaths.Temp, DirectoryPaths.SetupSnapshot); diff --git a/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs b/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs index 09d8f8295..1b850aeb3 100644 --- a/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs +++ b/Lombiq.Tests.UI/Services/OrchardCoreUITestExecutorConfiguration.cs @@ -1,14 +1,17 @@ +using Lombiq.Tests.UI.Constants; using Lombiq.Tests.UI.Extensions; using Lombiq.Tests.UI.Helpers; using Lombiq.Tests.UI.SecurityScanning; using Lombiq.Tests.UI.Services.GitHub; using Lombiq.Tests.UI.SqlQueryMonitoring.Services; +using Microsoft.Extensions.Logging; using OpenQA.Selenium.BiDi.Log; using OpenQA.Selenium.BiDi.Network; using Shouldly; using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.IO; using System.Threading; using System.Threading.Tasks; using Xunit; @@ -31,28 +34,35 @@ public enum Browser public class OrchardCoreUITestExecutorConfiguration { - public static readonly Func AssertAppLogsAreEmptyAsync = app => - app.LogsShouldBeEmptyAsync(TestContext.Current.CancellationToken); + public static readonly Func AssertAppLogsAreEmptyAsync = + app => app.LogsShouldBeEmptyAsync(TestContext.Current.CancellationToken); + public static readonly Func AssertAppLogsCanContainFeatureSkipAsync = + app => app.LogsShouldNotContainAsync( + entry => + entry.Level >= LogLevel.Warning && + entry.Message != "Skipping feature 'OrchardCore.Tenants' as it is allowed on the default tenant only.", + TestContext.Current.CancellationToken); + + [Obsolete("This is no longer necessary after https://github.com/OrchardCMS/OrchardCore/pull/18341.")] public static readonly Func AssertAppLogsCanContainCacheFolderErrorsAsync = app => app.LogsShouldNotContainAsync(AppLogAssertionHelper.NotMediaCacheEntriesPredicate, TestContext.Current.CancellationToken); - public static readonly Action> AssertBrowserLogIsEmpty = + public static readonly Action> AssertBrowserLogIsEmpty = logEntries => logEntries.ShouldBeEmpty(logEntries.ToFormattedString()); + /// + /// The default browser log filter. Ignores logs below . + /// public static readonly Func IsNonSuccessBrowserLogEntry = - entry => - entry.Level >= Level.Warn && - // HTML imports are somehow used by Selenium or something but this deprecation notice is always there for - // every page. - !entry.Text.ContainsOrdinalIgnoreCase("HTML Imports is deprecated"); + entry => entry.Level >= Level.Warn; // The 404 is because of how browsers automatically request /favicon.ico even if a favicon is declared to be under a // different URL. public static readonly Func IsNonSuccessResponse = e => e.Response.Status is < 200 or >= 400 && !e.Response.Url.EndsWithOrdinalIgnoreCase("/favicon.ico"); - public static readonly Action> AssertResponseLogIsEmpty = + public static readonly Action> AssertResponseLogIsEmpty = responses => responses.ShouldBeEmpty(responses.ToFormattedString()); private CancellationToken _testCancellationToken; @@ -86,7 +96,7 @@ public class OrchardCoreUITestExecutorConfiguration $"{nameof(OrchardCoreUITestExecutorConfiguration)}:RetryIntervalSeconds", 0)); - public Func AssertAppLogsAsync { get; set; } = AssertAppLogsCanContainCacheFolderErrorsAsync; + public Func AssertAppLogsAsync { get; set; } = AssertAppLogsCanContainFeatureSkipAsync; /// /// Gets a collection of delegate that selects which response data get saved to BrowserLogFilter set => BrowserLogFilters[nameof(BrowserLogFilter)] = value; } - public Action> AssertBrowserLog { get; set; } = AssertBrowserLogIsEmpty; + public Action> AssertBrowserLog { get; set; } = AssertBrowserLogIsEmpty; /// - /// Gets or sets a delegate that selects which response data get saved to . /// - public Func ResponseLogFilter { get; set; } = IsNonSuccessResponse; + public IDictionary> ResponseLogFilters { get; } = + new Dictionary> + { + [nameof(IsNonSuccessResponse)] = IsNonSuccessResponse, + ["Ignore missing favicon."] = eventArgs => !eventArgs.Response.Url.ContainsOrdinalIgnoreCase("favicon.ico"), + }; - public Action> AssertResponseLog { get; set; } = AssertResponseLogIsEmpty; + /// + /// Gets or sets the default delegate that selects which response data get saved to . It's a shortcut to the entry + /// where the key is . + /// + [Obsolete($"Use {nameof(ResponseLogFilters)} directly.")] + public Func ResponseLogFilter + { + get => ResponseLogFilters.GetMaybe(nameof(ResponseLogFilter)) ?? (_ => true); + set => ResponseLogFilters[nameof(ResponseLogFilter)] = value; + } + + public Action> AssertResponseLog { get; set; } = AssertResponseLogIsEmpty; public ITestOutputHelper TestOutputHelper { get; set; } @@ -214,4 +241,31 @@ public CancellationToken TestCancellationToken get => _testCancellationToken == default ? TestContext.Current.CancellationToken : _testCancellationToken; set => _testCancellationToken = value; } + + /// + /// Gets or sets the value which will be copied into upon creation. + /// + public string TempDirectoryPath { get; set; } + + /// + /// Gets if it's not , otherwise gets the path of the in the current directory. + /// + public string GetTempDirectoryPathWithFallback(params string[] subDirectories) => + GetTempDirectoryPathWithFallback(TempDirectoryPath, subDirectories); + + /// + /// Gets if it's not , otherwise gets the path of the + /// in the current directory. + /// + public static string GetTempDirectoryPathWithFallback(string tempDirectoryPath, params string[] subDirectories) + { + var path = string.IsNullOrEmpty(tempDirectoryPath) + ? Path.Combine(Environment.CurrentDirectory, DirectoryPaths.Temp) + : tempDirectoryPath; + + if (subDirectories.Length > 0) path = Path.Combine([path, .. subDirectories]); + + return path; + } } diff --git a/Lombiq.Tests.UI/Services/UITestContext.cs b/Lombiq.Tests.UI/Services/UITestContext.cs index 2ddbfb25d..45c30d75d 100644 --- a/Lombiq.Tests.UI/Services/UITestContext.cs +++ b/Lombiq.Tests.UI/Services/UITestContext.cs @@ -1,4 +1,5 @@ using Atata; +using Lombiq.HelpfulLibraries.Common.Utilities; using Lombiq.Tests.UI.Constants; using Lombiq.Tests.UI.Exceptions; using Lombiq.Tests.UI.Extensions; @@ -12,6 +13,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -152,7 +154,7 @@ public sealed class UITestContext : IAsyncDisposable /// Gets the current tenant name. When testing sites with multi-tenancy use /// . /// - public string TenantName { get; private set; } = "Default"; + public string TenantName { get; private set; } = ShellSettings.DefaultShellName; /// /// Gets or sets the prefix used for all relative URLs. It should neither start nor end with a slash. @@ -166,6 +168,12 @@ public sealed class UITestContext : IAsyncDisposable /// public string AdminUrlPrefix { get; set; } = "/Admin"; + /// + /// Gets the path of the directory where running instances are stored. If + /// the internal value is set to (which is the default), then ./Temp/ is used. + /// + public string TempDirectoryPath { get; } + /// /// Gets the absolute path of the subdirectory inside the current test /// instance's directory. @@ -206,6 +214,8 @@ private UITestContext( AzureBlobStorageRunningContext, ElasticsearchRunningContext ) = runningContextContainer; + + TempDirectoryPath = configuration.GetTempDirectoryPathWithFallback(); } /// @@ -314,6 +324,7 @@ public static async Task CreateAsync( ZapManager zapManager) #pragma warning restore S107 // Methods should not have too many parameters { + var token = configuration.TestCancellationToken; var context = new UITestContext( id, testManifest, @@ -324,31 +335,37 @@ public static async Task CreateAsync( runningContextContainer, zapManager); + FileSystemHelper.EnsureDirectoryExists(context.TempDirectoryPath); + if (context.IsBrowserConfigured) { - context._biDirectionalDriver = await scope.Driver.AsBiDiAsync(); + context._biDirectionalDriver = await scope.Driver.AsBiDiAsync(cancellationToken: token); // We intentionally don't pass the UITestContext to these callbacks: The callbacks are called asynchronously // by the browser (and Selenium), and e.g. the current URL can change between when a JS exception was thrown // and the callback is called. Thus, BrowserLogFilter could e.g. ignore log entries for a URL that actually // originated from a different URL and shouldn't be ignored. - await context._biDirectionalDriver.Log.OnEntryAddedAsync(entry => - { - if (configuration.BrowserLogFilters.Values.All(filter => filter(entry))) + await context._biDirectionalDriver.Log.OnEntryAddedAsync( + entry => { - context._cumulativeBrowserLog.Enqueue(entry); - } - }); + if (configuration.BrowserLogFilters.Values.All(filter => filter(entry))) + { + context._cumulativeBrowserLog.Enqueue(entry); + } + }, + cancellationToken: token); if (configuration.TestDumpConfiguration.CaptureResponseLog) { - await context._biDirectionalDriver.Network.OnResponseCompletedAsync(responseCompleted => - { - if (configuration.ResponseLogFilter(responseCompleted)) + await context._biDirectionalDriver.Network.OnResponseCompletedAsync( + responseCompleted => { - context._cumulativeResponseLog.Enqueue(responseCompleted.Response); - } - }); + if (configuration.ResponseLogFilters.All(filter => filter.Value(responseCompleted))) + { + context._cumulativeResponseLog.Enqueue(responseCompleted.Response); + } + }, + cancellationToken: token); } } @@ -356,18 +373,18 @@ await context._biDirectionalDriver.Network.OnResponseCompletedAsync(responseComp } /// - /// Returns the subdirectory described by inside the current test instance's - /// directory. + /// Returns the subdirectory described by the sub-path inside the / directory. /// public string GetTempSubDirectoryPath(params string[] subDirectoryNames) => - DirectoryPaths.GetTempDirectoryPath([Id, .. subDirectoryNames]); + Path.Combine([TempDirectoryPath, Id, .. subDirectoryNames]); /// /// Returns a path in the subdirectory inside the current test instance's /// directory. /// public string GetDownloadFilePath(params string[] subDirectoryNames) => - DirectoryPaths.GetTempDirectoryPath([Id, DirectoryPaths.Downloads, .. subDirectoryNames]); + Path.Combine([TempDirectoryPath, Id, DirectoryPaths.Downloads, .. subDirectoryNames]); private bool IsAlert() { diff --git a/Lombiq.Tests.UI/Services/UITestExecutionSession.cs b/Lombiq.Tests.UI/Services/UITestExecutionSession.cs index 1adf009ad..d85e73b4d 100644 --- a/Lombiq.Tests.UI/Services/UITestExecutionSession.cs +++ b/Lombiq.Tests.UI/Services/UITestExecutionSession.cs @@ -202,6 +202,8 @@ await CreateTestDumpAsync( private async ValueTask ShutdownAsync() { + var tempSubDirectoryPath = _context?.GetTempSubDirectoryPath(); + _testOutputHelper.WriteLineTimestampedAndDebug("Shutting down the test execution session."); if (_configuration.RunAssertLogsOnAllPageChanges) @@ -218,16 +220,15 @@ private async ValueTask ShutdownAsync() _configuration.Events.AfterClick -= TakeScreenshotIfEnabledAsync; } - if (_applicationInstance != null) await _applicationInstance.DisposeAsync(); - - string contextId = null; - - if (_context != null) + if (_context?.ElasticsearchRunningContext is { } elasticsearchRunningContext) { - contextId = _context.Id; - await _context.DisposeAsync(); + await elasticsearchRunningContext.AfterTestAsync(_context); } + if (_applicationInstance != null) await _applicationInstance.DisposeAsync(); + + if (_context != null) await _context.DisposeAsync(); + if (_sqlServerManager is not null) { await _sqlServerManager.DisposeAsync(); @@ -241,12 +242,14 @@ private async ValueTask ShutdownAsync() // handles to the temp folder, that can be cleaned up too. No need to do it on ephemeral GitHub runners, though, // also because the ZAP report's folder (like "2025-01-22-ZAP-Report-localhost") will remain unwritable (see the // comment in ZapManager). - if (!string.IsNullOrEmpty(contextId) && !GitHubHelper.IsGitHubEnvironment) + if (!string.IsNullOrEmpty(tempSubDirectoryPath) && + Directory.Exists(tempSubDirectoryPath) && + !GitHubHelper.IsGitHubEnvironment) { try { // This is a clean-up method, no need to forward a CancellationToken. - await DirectoryHelper.SafelyDeleteDirectoryIfExistsAsync(DirectoryPaths.GetTempDirectoryPath(contextId), CancellationToken.None); + await DirectoryHelper.SafelyDeleteDirectoryIfExistsAsync(tempSubDirectoryPath, CancellationToken.None); } catch (Exception ex) when (GitHubHelper.IsGitHubEnvironment) { @@ -258,11 +261,6 @@ private async ValueTask ShutdownAsync() } } - if (_context?.ElasticsearchRunningContext is { } elasticsearchRunningContext) - { - await elasticsearchRunningContext.AfterTestAsync(_context); - } - _screenshotCount = 0; _context = null; @@ -691,8 +689,7 @@ private async Task CreateContextAsync(Uri testStartRelativeUri) { var contextId = Guid.NewGuid().ToString(); _configuration.BrowserConfiguration.UITestContextId = contextId; - - FileSystemHelper.EnsureDirectoryExists(DirectoryPaths.GetTempDirectoryPath(contextId)); + _configuration.BrowserConfiguration.TempDirectoryPath = _configuration.TempDirectoryPath; var sqlServerContext = _configuration.UseSqlServer ? await SetUpSqlServerAsync() : null; var azureBlobStorageContext = _configuration.UseAzureBlobStorage ? await SetUpAzureBlobStorageAsync() : null; @@ -703,6 +700,7 @@ private async Task CreateContextAsync(Uri testStartRelativeUri) Task UITestingBeforeAppStartHandlerAsync(OrchardCoreAppStartContext context, InstanceCommandLineArgumentsBuilder arguments) { + arguments.AddWithValue("OrchardCore:OrchardCore_YesSql:EnableThreadSafetyChecks", value: true); arguments.AddWithValue("Lombiq_Tests_UI:IsUITesting", value: true); arguments.AddWithValue( "Lombiq_Tests_UI:EnableSqlQueryMonitoring", @@ -890,7 +888,7 @@ private async Task StartSmtpServiceAsync() (entry.Text.ContainsOrdinalIgnoreCase("ace-builds/src-noconflict/worker-html.js") || entry.StackTrace?.CallFrames.FirstOrDefault()?.Url.ContainsOrdinalIgnoreCase(smtpContext.WebUIUri.AbsoluteUri) == true)); - _configuration.ResponseLogFilter = e => e.IsNonSuccessResponseAndNotExpectedNotFoundResponse("/worker-html.js"); + _configuration.WithIgnoreExpectedNotFoundResponseFilter("/worker-html.js"); Task SmtpServiceBeforeAppStartHandlerAsync(OrchardCoreAppStartContext context, InstanceCommandLineArgumentsBuilder arguments) { @@ -914,12 +912,11 @@ Task SmtpServiceBeforeAppStartHandlerAsync(OrchardCoreAppStartContext context, I private ElasticsearchRunningContext SetUpElasticsearch() { - var id = Guid.NewGuid(); - var prefix = TestContext.Current.GetElasticsearchSafeIndexName(id); + var prefix = TestContext.Current.GetElasticsearchSafeIndexName(); _configuration.OrchardCoreConfiguration.ConfigureElasticsearchPrefix(prefix); - return new(id, prefix); + return new(prefix); } private async Task CaptureBrowserUsingDumpsAsync(string debugInformationPath) diff --git a/Lombiq.Tests.UI/Services/WebDriverFactory.cs b/Lombiq.Tests.UI/Services/WebDriverFactory.cs index 984dc907f..963e1918e 100644 --- a/Lombiq.Tests.UI/Services/WebDriverFactory.cs +++ b/Lombiq.Tests.UI/Services/WebDriverFactory.cs @@ -331,7 +331,7 @@ private static TDriverOptions SetCommonChromiumOptions( if (configuration.FakeVideoSource is not null) { - var fakeCameraSourceFilePath = configuration.FakeVideoSource.SaveVideoToTempFolder(); + var fakeCameraSourceFilePath = configuration.FakeVideoSource.SaveVideoToTempFolder(configuration.TempDirectoryPath); // In some cases the video would not start automatically. To avoid this scenario we are adding the // "disable-gesture-requirement-for-media-playback" flag. @@ -391,7 +391,10 @@ private static async Task> CreateDriverAsync(Func GetSqlQueryMonitoringStoreAs { ISqlQueryMonitoringStore store = null; - await context.Application.UsingScopeAsync( + await context.Application.UsingScopeServiceProviderAsync( serviceProvider => { store = serviceProvider.GetService(); diff --git a/Lombiq.Tests.UI/SqlQueryMonitoring/SqlQueryExecutionEntry.cs b/Lombiq.Tests.UI/SqlQueryMonitoring/SqlQueryExecutionEntry.cs index e78c3e9c3..128f548cb 100644 --- a/Lombiq.Tests.UI/SqlQueryMonitoring/SqlQueryExecutionEntry.cs +++ b/Lombiq.Tests.UI/SqlQueryMonitoring/SqlQueryExecutionEntry.cs @@ -53,7 +53,7 @@ private static string NormalizeCommandText(string commandText) /// E.g. "SELECT *" and "SELECT *" should count as the same query text. /// private static string NormalizeWhitespace(string text) => - WhitespaceRegex().Replace(text ?? string.Empty, " ").Trim(); + WhitespaceRegex.Replace(text ?? string.Empty, " ").Trim(); private static string CaptureCallStack() { @@ -95,5 +95,5 @@ private static string NormalizeParameterValue(object value) } [GeneratedRegex(@"\s+", RegexOptions.CultureInvariant, matchTimeoutMilliseconds: 1000)] - private static partial Regex WhitespaceRegex(); + private static partial Regex WhitespaceRegex { get; } } diff --git a/Lombiq.UITestingToolbox.sln b/Lombiq.UITestingToolbox.sln deleted file mode 100644 index 1cedefbb0..000000000 --- a/Lombiq.UITestingToolbox.sln +++ /dev/null @@ -1,37 +0,0 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.0.32112.339 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Lombiq.Tests.UI", "Lombiq.Tests.UI\Lombiq.Tests.UI.csproj", "{DFD31967-1510-4C93-A7F9-E55C307CD230}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Lombiq.Tests.UI.AppExtensions", "Lombiq.Tests.UI.AppExtensions\Lombiq.Tests.UI.AppExtensions.csproj", "{4E000537-7F13-4145-AA8F-DDDECBEAEC9F}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Lombiq.Tests.UI.Shortcuts", "Lombiq.Tests.UI.Shortcuts\Lombiq.Tests.UI.Shortcuts.csproj", "{8D9BE6BF-63D3-4329-803A-DCFE8CA94D54}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Any CPU = Debug|Any CPU - Release|Any CPU = Release|Any CPU - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {DFD31967-1510-4C93-A7F9-E55C307CD230}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {DFD31967-1510-4C93-A7F9-E55C307CD230}.Debug|Any CPU.Build.0 = Debug|Any CPU - {DFD31967-1510-4C93-A7F9-E55C307CD230}.Release|Any CPU.ActiveCfg = Release|Any CPU - {DFD31967-1510-4C93-A7F9-E55C307CD230}.Release|Any CPU.Build.0 = Release|Any CPU - {4E000537-7F13-4145-AA8F-DDDECBEAEC9F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {4E000537-7F13-4145-AA8F-DDDECBEAEC9F}.Debug|Any CPU.Build.0 = Debug|Any CPU - {4E000537-7F13-4145-AA8F-DDDECBEAEC9F}.Release|Any CPU.ActiveCfg = Release|Any CPU - {4E000537-7F13-4145-AA8F-DDDECBEAEC9F}.Release|Any CPU.Build.0 = Release|Any CPU - {8D9BE6BF-63D3-4329-803A-DCFE8CA94D54}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {8D9BE6BF-63D3-4329-803A-DCFE8CA94D54}.Debug|Any CPU.Build.0 = Debug|Any CPU - {8D9BE6BF-63D3-4329-803A-DCFE8CA94D54}.Release|Any CPU.ActiveCfg = Release|Any CPU - {8D9BE6BF-63D3-4329-803A-DCFE8CA94D54}.Release|Any CPU.Build.0 = Release|Any CPU - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection - GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {030E57A3-0EFB-4B35-8587-328927F03BD2} - EndGlobalSection -EndGlobal diff --git a/Lombiq.UITestingToolbox.slnx b/Lombiq.UITestingToolbox.slnx new file mode 100644 index 000000000..1666e233c --- /dev/null +++ b/Lombiq.UITestingToolbox.slnx @@ -0,0 +1,5 @@ + + + + + diff --git a/renovate.json5 b/renovate.json5 index 76d8686f8..b71cb169e 100644 --- a/renovate.json5 +++ b/renovate.json5 @@ -58,10 +58,6 @@ prBodyNotes: [ 'For details about the `rnwood.smtp4dev` release, see [its release notes](https://github.com/rnwood/smtp4dev/releases/tag/{{newVersion}}).', ], - // smtp4dev 3.15.0 switched from net8.0 to net10.0, breaking dotnet tool restore on .NET 8 SDK (CI uses - // 8.0.x). The DotnetToolSettings.xml is only under tools/net10.0/any/ so the .NET 8 SDK can't find it. - // Re-enable once the project and CI migrate to .NET 10. - enabled: false, }, { groupName: 'ZAP',