From fe4f95279779f86e9c565edf455575a971e6b0b7 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Sat, 7 Mar 2026 22:32:58 +0100 Subject: [PATCH 1/7] starting with Overlay --- core-parent/pom.xml | 5 +++++ core/pom.xml | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/core-parent/pom.xml b/core-parent/pom.xml index c57c828cb0..a4de02a50e 100644 --- a/core-parent/pom.xml +++ b/core-parent/pom.xml @@ -67,6 +67,11 @@ swagger-request-validator-core 2.44.9 + + com.webfuzzing + overlay-jvm + 0.1.1 + diff --git a/core/pom.xml b/core/pom.xml index e2097fe2bf..9d2cd099aa 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -39,6 +39,10 @@ com.webfuzzing commons + + com.webfuzzing + overlay-jvm + From b462cf662f9e27a0d8e847aea109c5ee5535fec2 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Wed, 11 Mar 2026 15:03:56 +0100 Subject: [PATCH 2/7] config options for Overlay --- .../kotlin/org/evomaster/core/EMConfig.kt | 54 +++++++++++++++++-- docs/options.md | 3 ++ 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index 6bd096e17f..ad9b902aea 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -634,7 +634,6 @@ class EMConfig { throw ConfigProblemException("'bbTargetUrl' should be set only in black-box mode") } - // ONUR, this line is changed since it did not compile in the previous case. if (!endpointFocus.isNullOrBlank() && !endpointPrefix.isNullOrBlank()) { throw ConfigProblemException("both 'endpointFocus' and 'endpointPrefix' are set") } @@ -913,7 +912,7 @@ class EMConfig { if (Files.exists(path) && !Files.isWritable(path)) { throw ConfigProblemException("Parameter '${m.name}' refers to a file that already" + - " exists, but that cannot be written/replace to: $path") + " exists, but that cannot be written/replaced to: $path") } if (Files.exists(path) && Files.isDirectory(path)) { throw ConfigProblemException("Parameter '${m.name}' refers to a file that is instead an" + @@ -921,6 +920,26 @@ class EMConfig { } } } + + m.annotations.find { it is ExistingPath }?.also{ + val ep = it as ExistingPath + if(! (ep.canBeBlank && parameterValue.isBlank())){ + val path = try { + Paths.get(parameterValue).toAbsolutePath() + } catch (e: InvalidPathException) { + throw ConfigProblemException("Parameter '${m.name}' is not a valid FS path: ${e.message}") + } + + if (!Files.exists(path)){ + throw ConfigProblemException("File/folder for '${m.name}' does not exist: $path") + } + + if( ep.shouldBeWritable && !Files.isWritable(path)){ + throw ConfigProblemException("Parameter '${m.name}' refers to a file that" + + " exists, but it cannot be written/replaced to: $path") + } + } + } } @@ -1052,7 +1071,7 @@ class EMConfig { /** - * This represent one of the main properties to set in EvoMaster. + * This represents one of the main properties to set in EvoMaster. * Those are the ones most likely going to be set by practitioners. * Note: most of the other properties are mainly for experiments */ @@ -1074,6 +1093,13 @@ class EMConfig { @MustBeDocumented annotation class FilePath(val canBeBlank: Boolean = false) + /** + * Either a file or a folder, that MUST already exist and can be read. + */ + @Target(AnnotationTarget.PROPERTY) + @MustBeDocumented + annotation class ExistingPath(val canBeBlank: Boolean = false, val shouldBeWritable: Boolean = false) + //------------------------------------------------------------------------ @@ -3049,6 +3075,28 @@ class EMConfig { " path element of the URL will not change).") var overrideAuthExternalEndpointURL : String? = null + @Experimental + @ExistingPath(true,false) + @Cfg("Specify an OAI Overlay file, or a folder containing those." + + " In this latter case, Overlay files will be searched recursively in the nested folder, matching" + + " a given list of configurable suffixes." + + " Each Overlay will be applied to the target OpenAPI schema." + + " If more than one Overlay file is applied, no specific ordering of transformations is enforced.") + var overlay = "" + + @Experimental + @Cfg("Comma ',' separated list of file name suffixes." + + " When scanning a folder for OAI Overlay files, any file with name matching any one of these" + + " suffixes will be loaded and applied.") + var overlayFileSuffixes = ".json,.yaml,.yml" + + @Experimental + @Cfg("When applying Overlay transformations, by default EvoMaster will crash immediately" + + " if there is any issue with the transformations, e.g., if some transformations are not applied" + + " because the JSON Path selectors found no applicable node in the OpenAPI schema." + + " This option can be used to override such behavior, and let the fuzzing go on without" + + " applying any overlay.") + var overlayLenient = false fun getProbabilityUseDataPool() : Double{ return if(blackBox){ diff --git a/docs/options.md b/docs/options.md index 7f5c5369b0..091ddcc843 100644 --- a/docs/options.md +++ b/docs/options.md @@ -300,6 +300,9 @@ There are 3 types of options: |`maxTestSizeStrategy`| __Enum__. Specify a strategy to handle a max size of a test. *Valid values*: `SPECIFIED, DPC_INCREASING, DPC_DECREASING`. *Default value*: `SPECIFIED`.| |`mutationTargetsSelectionStrategy`| __Enum__. Specify a strategy to select targets for evaluating mutation. *Valid values*: `FIRST_NOT_COVERED_TARGET, EXPANDED_UPDATED_NOT_COVERED_TARGET, UPDATED_NOT_COVERED_TARGET`. *Default value*: `FIRST_NOT_COVERED_TARGET`.| |`onePlusLambdaLambdaOffspringSize`| __Int__. 1+(λ,λ) GA: number of offspring (λ) per generation. *Constraints*: `min=1.0`. *Default value*: `4`.| +|`overlay`| __String__. Specify an OAI Overlay file, or a folder containing those. In this latter case, Overlay files will be searched recursively in the nested folder, matching a given list of configurable suffixes. Each Overlay will be applied to the target OpenAPI schema. If more than one Overlay file is applied, no specific ordering of transformations is enforced. *Default value*: `""`.| +|`overlayFileSuffixes`| __String__. Comma ',' separated list of file name suffixes. When scanning a folder for OAI Overlay files, any file with name matching any one of these suffixes will be loaded and applied. *Default value*: `.json,.yaml,.yml`.| +|`overlayLenient`| __Boolean__. When applying Overlay transformations, by default EvoMaster will crash immediately if there is any issue with the transformations, e.g., if some transformations are not applied because the JSON Path selectors found no applicable node in the OpenAPI schema. This option can be used to override such behavior, and let the fuzzing go on without applying any overlay. *Default value*: `false`.| |`prematureStopStrategy`| __Enum__. Specify how 'improvement' is defined: either any kind of improvement even if partial (ANY), or at least one new target is fully covered (NEW). *Valid values*: `ANY, NEW`. *Default value*: `NEW`.| |`probOfHandlingLength`| __Double__. Specify a probability of applying length handling. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.0`.| |`probOfHarvestingResponsesFromActualExternalServices`| __Double__. a probability of harvesting actual responses from external services as seeds. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.0`.| From 3a8264c75f3a6bf58494aee008be79a26cfac347 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Wed, 11 Mar 2026 22:25:19 +0100 Subject: [PATCH 3/7] applying Overlays --- .../kotlin/org/evomaster/core/EMConfig.kt | 2 +- .../core/problem/rest/schema/SchemaOpenAPI.kt | 49 ++++++++++++++++++- .../core/problem/rest/schema/SchemaUtils.kt | 37 ++++++++++++++ .../service/sampler/AbstractRestSampler.kt | 31 +++++++++++- docs/options.md | 2 +- 5 files changed, 115 insertions(+), 6 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index ad9b902aea..05de764672 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -3077,7 +3077,7 @@ class EMConfig { @Experimental @ExistingPath(true,false) - @Cfg("Specify an OAI Overlay file, or a folder containing those." + + @Cfg("Specify an OAI Overlay file path, or a folder containing those." + " In this latter case, Overlay files will be searched recursively in the nested folder, matching" + " a given list of configurable suffixes." + " Each Overlay will be applied to the target OpenAPI schema." + diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/schema/SchemaOpenAPI.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/schema/SchemaOpenAPI.kt index 0c26073caa..ea1785eefe 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/schema/SchemaOpenAPI.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/schema/SchemaOpenAPI.kt @@ -1,12 +1,14 @@ package org.evomaster.core.problem.rest.schema +import com.webfuzzing.overlayjvm.Processor import io.swagger.v3.oas.models.OpenAPI +import org.slf4j.LoggerFactory class SchemaOpenAPI( /** * The actual raw value of the schema file, as a string */ - val schemaRaw : String, + val schemaRaw: String, /** * A parsed schema */ @@ -16,4 +18,47 @@ class SchemaOpenAPI( * It can be null, eg, if provided as string in our own EM tests */ val sourceLocation: SchemaLocation -) \ No newline at end of file +) { + + companion object { + private val log = LoggerFactory.getLogger(SchemaOpenAPI::class.java) + } + + + fun withOverlays(overlays: List?, lenient: Boolean): SchemaOpenAPI { + if (overlays.isNullOrEmpty()) { + return this + } + + var modified = schemaRaw + overlays.forEach { + val res = try{ + Processor.applyOverlay(modified,it) + } catch (e : Exception){ + val message = "Failed to apply overlay transformation: ${e.message}" + if(lenient){ + log.warn(message) + return@forEach + } else { + throw IllegalArgumentException(message,e) + } + } + + if(res.warnings.isNotEmpty()){ + val message = "Issues (${res.warnings.size}) with applying Overlay transformation:\n${res.warnings.joinToString("\n")}" + if(lenient){ + log.warn(message) + //here we do not return, as we still apply the partial transformations + } else { + throw IllegalArgumentException(message) + } + } + + modified = res.transformedSchema + } + + return OpenApiAccess.parseOpenApi(modified, this.sourceLocation) + } + + +} \ No newline at end of file diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/schema/SchemaUtils.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/schema/SchemaUtils.kt index ec19c3f3fc..0eaa3b7beb 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/schema/SchemaUtils.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/schema/SchemaUtils.kt @@ -10,6 +10,12 @@ import io.swagger.v3.oas.models.parameters.RequestBody import io.swagger.v3.oas.models.responses.ApiResponse import java.net.URI import java.net.URISyntaxException +import java.nio.file.Files +import java.nio.file.Paths +import kotlin.io.path.exists +import kotlin.io.path.name +import kotlin.io.path.readText +import kotlin.io.path.walk /** * https://swagger.io/docs/specification/v3_0/using-ref/ @@ -256,4 +262,35 @@ object SchemaUtils { return response } + + /** + * depending on whether path is a file or folder, return a list with 1 or more Overlays. + * Suffix check is only done if folder. + * If path is empty, return null. + */ + fun readOverlayFiles(path: String, suffixes: String): List? { + + if(path.isBlank()){ + //nothing to do + return null + } + + val ap = Paths.get(path).toAbsolutePath() + + if(!ap.exists()){ + throw IllegalArgumentException("Path $path does not point to any existing file or folder") + } + + if(Files.isRegularFile(ap)){ + //only one file + return listOf(ap.readText()) + } + + val options = suffixes.split(',').map { it.trim() } + + return ap.walk() + .filter{file -> options.any{s -> file.name.endsWith(s) } } + .map { it.readText() } + .toList() + } } \ No newline at end of file diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/sampler/AbstractRestSampler.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/sampler/AbstractRestSampler.kt index 57df7a1083..344f2546bb 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/sampler/AbstractRestSampler.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/sampler/AbstractRestSampler.kt @@ -6,6 +6,7 @@ import org.evomaster.client.java.controller.api.dto.problem.ExternalServiceDto import org.evomaster.client.java.instrumentation.shared.TaintInputName import org.evomaster.core.AnsiColor import org.evomaster.core.EMConfig +import org.evomaster.core.config.ConfigProblemException import org.evomaster.core.logging.LoggingUtil import org.evomaster.core.output.service.PartialOracles import org.evomaster.core.problem.enterprise.SampleType @@ -26,6 +27,8 @@ import org.evomaster.core.problem.rest.param.QueryParam import org.evomaster.core.problem.rest.schema.OpenApiAccess import org.evomaster.core.problem.rest.schema.RestSchema import org.evomaster.core.problem.rest.schema.SchemaLocation +import org.evomaster.core.problem.rest.schema.SchemaOpenAPI +import org.evomaster.core.problem.rest.schema.SchemaUtils import org.evomaster.core.problem.rest.seeding.Parser import org.evomaster.core.problem.rest.seeding.postman.PostmanParser import org.evomaster.core.problem.rest.service.AIResponseClassifier @@ -110,7 +113,9 @@ abstract class AbstractRestSampler : HttpWsSampler() { } else { throw SutProblemException("No info on the OpenAPI schema was provided") } - schemaHolder = RestSchema(swagger) + val transformed = applyOverlays(swagger) + + schemaHolder = RestSchema(transformed) schemaHolder.validate() // The code should never reach this line without a valid swagger. @@ -281,6 +286,27 @@ abstract class AbstractRestSampler : HttpWsSampler() { } + private fun retrieveOverlays() : List? { + val overlays = SchemaUtils.readOverlayFiles(config.overlay, config.overlayFileSuffixes) + + if(overlays!=null && overlays.isEmpty()){ + throw ConfigProblemException("Could not find any Overlay file in '${config.overlay}' given the suffixes '${config.overlayFileSuffixes}'") + } + + return overlays + } + + private fun applyOverlays(openApi: SchemaOpenAPI) : SchemaOpenAPI{ + + val overlays = retrieveOverlays() + + val transformed = try{ + openApi.withOverlays(overlays, config.overlayLenient) + }catch (e: Exception){ + throw ConfigProblemException(e.message ?: "Failed to apply overlays") + } + return transformed + } private fun initForBlackBox() { @@ -288,8 +314,9 @@ abstract class AbstractRestSampler : HttpWsSampler() { // retrieve the swagger val swagger = OpenApiAccess.getOpenAPIFromLocation(configuration.bbSwaggerUrl, authentications) + val transformed = applyOverlays(swagger) - schemaHolder = RestSchema(swagger) + schemaHolder = RestSchema(transformed) schemaHolder.validate() actionCluster.clear() diff --git a/docs/options.md b/docs/options.md index 091ddcc843..8b873456e7 100644 --- a/docs/options.md +++ b/docs/options.md @@ -300,7 +300,7 @@ There are 3 types of options: |`maxTestSizeStrategy`| __Enum__. Specify a strategy to handle a max size of a test. *Valid values*: `SPECIFIED, DPC_INCREASING, DPC_DECREASING`. *Default value*: `SPECIFIED`.| |`mutationTargetsSelectionStrategy`| __Enum__. Specify a strategy to select targets for evaluating mutation. *Valid values*: `FIRST_NOT_COVERED_TARGET, EXPANDED_UPDATED_NOT_COVERED_TARGET, UPDATED_NOT_COVERED_TARGET`. *Default value*: `FIRST_NOT_COVERED_TARGET`.| |`onePlusLambdaLambdaOffspringSize`| __Int__. 1+(λ,λ) GA: number of offspring (λ) per generation. *Constraints*: `min=1.0`. *Default value*: `4`.| -|`overlay`| __String__. Specify an OAI Overlay file, or a folder containing those. In this latter case, Overlay files will be searched recursively in the nested folder, matching a given list of configurable suffixes. Each Overlay will be applied to the target OpenAPI schema. If more than one Overlay file is applied, no specific ordering of transformations is enforced. *Default value*: `""`.| +|`overlay`| __String__. Specify an OAI Overlay file path, or a folder containing those. In this latter case, Overlay files will be searched recursively in the nested folder, matching a given list of configurable suffixes. Each Overlay will be applied to the target OpenAPI schema. If more than one Overlay file is applied, no specific ordering of transformations is enforced. *Default value*: `""`.| |`overlayFileSuffixes`| __String__. Comma ',' separated list of file name suffixes. When scanning a folder for OAI Overlay files, any file with name matching any one of these suffixes will be loaded and applied. *Default value*: `.json,.yaml,.yml`.| |`overlayLenient`| __Boolean__. When applying Overlay transformations, by default EvoMaster will crash immediately if there is any issue with the transformations, e.g., if some transformations are not applied because the JSON Path selectors found no applicable node in the OpenAPI schema. This option can be used to override such behavior, and let the fuzzing go on without applying any overlay. *Default value*: `false`.| |`prematureStopStrategy`| __Enum__. Specify how 'improvement' is defined: either any kind of improvement even if partial (ANY), or at least one new target is fully covered (NEW). *Valid values*: `ANY, NEW`. *Default value*: `NEW`.| From a0d577e0dd98ed803ae3eed51b32b2f0dd9632cf Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Thu, 12 Mar 2026 08:56:25 +0100 Subject: [PATCH 4/7] improved info to Overlay --- core/src/main/kotlin/org/evomaster/core/EMConfig.kt | 6 +++++- .../core/problem/rest/schema/SchemaOpenAPI.kt | 7 +++++-- .../evomaster/core/problem/rest/schema/SchemaUtils.kt | 10 +++++++++- docs/options.md | 2 +- 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index 05de764672..92b20d209b 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -3087,7 +3087,11 @@ class EMConfig { @Experimental @Cfg("Comma ',' separated list of file name suffixes." + " When scanning a folder for OAI Overlay files, any file with name matching any one of these" + - " suffixes will be loaded and applied.") + " suffixes will be loaded and applied." + + " For example, '.json' could be used to match all JSON files." + + " If the folder contains also other types of files with same extension, you might need to define" + + " some naming convention, and then use suffixes based on it, e.g., '-overlay.yaml' to match" + + " all YAML files whose name ends in 'overlay', like 'example-overlay.yaml'.") var overlayFileSuffixes = ".json,.yaml,.yml" @Experimental diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/schema/SchemaOpenAPI.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/schema/SchemaOpenAPI.kt index ea1785eefe..17210c6db2 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/schema/SchemaOpenAPI.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/schema/SchemaOpenAPI.kt @@ -30,6 +30,9 @@ class SchemaOpenAPI( return this } + val infoForLenient = "If this behavior is expected, you need to explicitly use '--overlayLenient true'" + + " to avoid EvoMaster from stopping." + var modified = schemaRaw overlays.forEach { val res = try{ @@ -40,7 +43,7 @@ class SchemaOpenAPI( log.warn(message) return@forEach } else { - throw IllegalArgumentException(message,e) + throw IllegalArgumentException("$message\n$infoForLenient",e) } } @@ -50,7 +53,7 @@ class SchemaOpenAPI( log.warn(message) //here we do not return, as we still apply the partial transformations } else { - throw IllegalArgumentException(message) + throw IllegalArgumentException("$message\n$infoForLenient") } } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/schema/SchemaUtils.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/schema/SchemaUtils.kt index 0eaa3b7beb..f044ec03db 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/schema/SchemaUtils.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/schema/SchemaUtils.kt @@ -8,10 +8,12 @@ import io.swagger.v3.oas.models.media.Schema import io.swagger.v3.oas.models.parameters.Parameter import io.swagger.v3.oas.models.parameters.RequestBody import io.swagger.v3.oas.models.responses.ApiResponse +import org.evomaster.core.logging.LoggingUtil import java.net.URI import java.net.URISyntaxException import java.nio.file.Files import java.nio.file.Paths +import kotlin.io.path.absolutePathString import kotlin.io.path.exists import kotlin.io.path.name import kotlin.io.path.readText @@ -283,14 +285,20 @@ object SchemaUtils { if(Files.isRegularFile(ap)){ //only one file + LoggingUtil.getInfoLogger().info("Retrieving Overlay from: $path") return listOf(ap.readText()) } val options = suffixes.split(',').map { it.trim() } + LoggingUtil.getInfoLogger().info("Scanning for Overlay files with possible suffix '$suffixes' in $path") + return ap.walk() .filter{file -> options.any{s -> file.name.endsWith(s) } } - .map { it.readText() } + .map { + LoggingUtil.getInfoLogger().info("Retrieving Overlay from: ${it.absolutePathString()}") + it.readText() + } .toList() } } \ No newline at end of file diff --git a/docs/options.md b/docs/options.md index 8b873456e7..57383b6e6b 100644 --- a/docs/options.md +++ b/docs/options.md @@ -301,7 +301,7 @@ There are 3 types of options: |`mutationTargetsSelectionStrategy`| __Enum__. Specify a strategy to select targets for evaluating mutation. *Valid values*: `FIRST_NOT_COVERED_TARGET, EXPANDED_UPDATED_NOT_COVERED_TARGET, UPDATED_NOT_COVERED_TARGET`. *Default value*: `FIRST_NOT_COVERED_TARGET`.| |`onePlusLambdaLambdaOffspringSize`| __Int__. 1+(λ,λ) GA: number of offspring (λ) per generation. *Constraints*: `min=1.0`. *Default value*: `4`.| |`overlay`| __String__. Specify an OAI Overlay file path, or a folder containing those. In this latter case, Overlay files will be searched recursively in the nested folder, matching a given list of configurable suffixes. Each Overlay will be applied to the target OpenAPI schema. If more than one Overlay file is applied, no specific ordering of transformations is enforced. *Default value*: `""`.| -|`overlayFileSuffixes`| __String__. Comma ',' separated list of file name suffixes. When scanning a folder for OAI Overlay files, any file with name matching any one of these suffixes will be loaded and applied. *Default value*: `.json,.yaml,.yml`.| +|`overlayFileSuffixes`| __String__. Comma ',' separated list of file name suffixes. When scanning a folder for OAI Overlay files, any file with name matching any one of these suffixes will be loaded and applied. For example, '.json' could be used to match all JSON files. If the folder contains also other types of files with same extension, you might need to define some naming convention, and then use suffixes based on it, e.g., '-overlay.yaml' to match all YAML files whose name ends in 'overlay', like 'example-overlay.yaml'. *Default value*: `.json,.yaml,.yml`.| |`overlayLenient`| __Boolean__. When applying Overlay transformations, by default EvoMaster will crash immediately if there is any issue with the transformations, e.g., if some transformations are not applied because the JSON Path selectors found no applicable node in the OpenAPI schema. This option can be used to override such behavior, and let the fuzzing go on without applying any overlay. *Default value*: `false`.| |`prematureStopStrategy`| __Enum__. Specify how 'improvement' is defined: either any kind of improvement even if partial (ANY), or at least one new target is fully covered (NEW). *Valid values*: `ANY, NEW`. *Default value*: `NEW`.| |`probOfHandlingLength`| __Double__. Specify a probability of applying length handling. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.0`.| From 4e0ad757c40a922c88379be67592ccd6bf3ad6d0 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Thu, 12 Mar 2026 10:18:17 +0100 Subject: [PATCH 5/7] clarification --- .../org/evomaster/core/problem/rest/schema/SchemaOpenAPI.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/schema/SchemaOpenAPI.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/schema/SchemaOpenAPI.kt index 17210c6db2..6a8efe370f 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/schema/SchemaOpenAPI.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/schema/SchemaOpenAPI.kt @@ -14,8 +14,8 @@ class SchemaOpenAPI( */ val schemaParsed: OpenAPI, /** - * the location the schema was retrieved from. - * It can be null, eg, if provided as string in our own EM tests + * information about the location the schema was retrieved from, e.g., + * from file, URL or in memory in our tests. */ val sourceLocation: SchemaLocation ) { From e909ab9ea5987f9f21d660c9d7b2a3b90cbf4b8a Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Thu, 12 Mar 2026 15:10:15 +0100 Subject: [PATCH 6/7] adding (failing) tests. need fix overlay-jvm --- .../openapi/v3/overlay/OverlayApplication.kt | 16 ++ .../spring/openapi/v3/overlay/OverlayRest.kt | 27 ++++ .../main/resources/overlay/subfolder/y.yaml | 10 ++ .../src/main/resources/overlay/x.yaml | 10 ++ .../src/main/resources/overlay/z.json | 19 +++ .../openapi/v3/overlay/OverlayController.kt | 5 + .../openapi/v3/overlay/OverlayEMTest.kt | 138 ++++++++++++++++++ 7 files changed, 225 insertions(+) create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/overlay/OverlayApplication.kt create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/overlay/OverlayRest.kt create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/resources/overlay/subfolder/y.yaml create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/resources/overlay/x.yaml create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/resources/overlay/z.json create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/overlay/OverlayController.kt create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/overlay/OverlayEMTest.kt diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/overlay/OverlayApplication.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/overlay/OverlayApplication.kt new file mode 100644 index 0000000000..15d07a54a1 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/overlay/OverlayApplication.kt @@ -0,0 +1,16 @@ +package com.foo.rest.examples.spring.openapi.v3.overlay + +import org.springframework.boot.SpringApplication +import org.springframework.boot.autoconfigure.SpringBootApplication +import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration + +@SpringBootApplication(exclude = [SecurityAutoConfiguration::class]) +open class OverlayApplication { + + companion object { + @JvmStatic + fun main(args: Array) { + SpringApplication.run(OverlayApplication::class.java, *args) + } + } +} \ No newline at end of file diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/overlay/OverlayRest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/overlay/OverlayRest.kt new file mode 100644 index 0000000000..4595091f6d --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/overlay/OverlayRest.kt @@ -0,0 +1,27 @@ +package com.foo.rest.examples.spring.openapi.v3.overlay + +import com.foo.rest.examples.spring.openapi.v3.charescaperegex.CharEscapeRegexDto +import com.foo.rest.examples.spring.openapi.v3.stringlength.StringLengthDto +import org.springframework.http.MediaType +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PathVariable +import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.RequestBody +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController +import java.util.regex.Pattern +import javax.validation.Valid +import javax.ws.rs.QueryParam + +@RestController +@RequestMapping(path = ["/api/overlay"]) +open class OverlayRest { + + @GetMapping + open fun get( @QueryParam("x") x: String, @QueryParam("y") y: String) : ResponseEntity{ + + return ResponseEntity.ok("x=$x , y=$y") + } + +} \ No newline at end of file diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/resources/overlay/subfolder/y.yaml b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/resources/overlay/subfolder/y.yaml new file mode 100644 index 0000000000..2cbc011215 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/resources/overlay/subfolder/y.yaml @@ -0,0 +1,10 @@ +overlay: 1.1.0 +info: + title: Add example to x parameter + version: 1.0.0 +actions: + - target: $.paths.['/api/overlay'].get.parameters[?(@.name == 'y')] + update: + examples: + foo: + value: "Some Y value like 777" \ No newline at end of file diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/resources/overlay/x.yaml b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/resources/overlay/x.yaml new file mode 100644 index 0000000000..53ea7053cd --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/resources/overlay/x.yaml @@ -0,0 +1,10 @@ +overlay: 1.1.0 +info: + title: Add example to x parameter + version: 1.0.0 +actions: + - target: $.paths.['/api/overlay'].get.parameters[?(@.name == 'x')] + update: + examples: + foo: + value: "Some X value like 1234" \ No newline at end of file diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/resources/overlay/z.json b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/resources/overlay/z.json new file mode 100644 index 0000000000..7cfaf2c952 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/resources/overlay/z.json @@ -0,0 +1,19 @@ +{ + "overlay": "1.1.0", + "info": { + "title": "Add example to non-existing z parameter", + "version": "1.0.0" + }, + "actions": [ + { + "target": "$.paths.['/api/overlay'].get.parameters[?(@.name == 'z')]", + "update": { + "examples": { + "foo": { + "value": "Some X value like 1234" + } + } + } + } + ] +} \ No newline at end of file diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/overlay/OverlayController.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/overlay/OverlayController.kt new file mode 100644 index 0000000000..2267f89d56 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/overlay/OverlayController.kt @@ -0,0 +1,5 @@ +package com.foo.rest.examples.spring.openapi.v3.overlay + +import com.foo.rest.examples.spring.openapi.v3.SpringController + +class OverlayController : SpringController(OverlayApplication::class.java) \ No newline at end of file diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/overlay/OverlayEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/overlay/OverlayEMTest.kt new file mode 100644 index 0000000000..801a9b8e04 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/overlay/OverlayEMTest.kt @@ -0,0 +1,138 @@ +package org.evomaster.e2etests.spring.openapi.v3.overlay + +import com.foo.rest.examples.spring.openapi.v3.overlay.OverlayController +import com.foo.rest.examples.spring.openapi.v3.stringlength.StringLengthController +import org.evomaster.core.config.ConfigProblemException +import org.evomaster.core.problem.rest.data.HttpVerb +import org.evomaster.e2etests.spring.openapi.v3.SpringTestBase +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows + +class OverlayEMTest : SpringTestBase(){ + + companion object { + @BeforeAll + @JvmStatic + fun init() { + initClass(OverlayController()) + } + } + + //these should be kept in sink with what written in the Overlay files + private val X = "Some X value like 1234" + private val Y = "Some Y value like 777" + + @Test + fun testRunEM_Overlay_None() { + + runTestHandlingFlakyAndCompilation( + "Overlay_None", + 100 + ) { args: MutableList -> + + val solution = initAndRun(args) + + Assertions.assertTrue(solution.individuals.size >= 1) + assertHasAtLeastOne(solution, HttpVerb.GET, 200, "/api/overlay", null) + assertNone(solution, HttpVerb.GET, 200, "/api/overlay", X) + assertNone(solution, HttpVerb.GET, 200, "/api/overlay", Y) + } + } + + + @Test + fun testRunEM_Overlay_X() { + + runTestHandlingFlakyAndCompilation( + "Overlay_X", + 100 + ) { args: MutableList -> + + setOption(args, "overlay", "src/main/resources/overlay/x.yaml") + + val solution = initAndRun(args) + + Assertions.assertTrue(solution.individuals.size >= 1) + assertHasAtLeastOne(solution, HttpVerb.GET, 200, "/api/overlay", X) + assertNone(solution, HttpVerb.GET, 200, "/api/overlay", Y) + } + } + + @Test + fun testRunEM_Overlay_Z_fail() { + + runTestHandlingFlakyAndCompilation( + "Overlay_Z_fail", + 100 + ) { args: MutableList -> + + setOption(args, "overlay", "src/main/resources/overlay/z.json") + //default behavior must be non-lenient + + //z does not exist + assertThrows { initAndRun(args) } + } + } + + @Test + fun testRunEM_Overlay_Z_lenient() { + + runTestHandlingFlakyAndCompilation( + "Overlay_Z_lenient", + 100 + ) { args: MutableList -> + + setOption(args, "overlay", "src/main/resources/overlay/z.json") + setOption(args, "overlayLenient", "true") + + //z does not exist... but, when lenient, shouldn't fail + val solution = initAndRun(args) + + Assertions.assertTrue(solution.individuals.size >= 1) + assertHasAtLeastOne(solution, HttpVerb.GET, 200, "/api/overlay", null) + assertNone(solution, HttpVerb.GET, 200, "/api/overlay", X) + assertNone(solution, HttpVerb.GET, 200, "/api/overlay", Y) + } + } + + @Test + fun testRunEM_Overlay_folder() { + + runTestHandlingFlakyAndCompilation( + "Overlay_folder", + 100 + ) { args: MutableList -> + + setOption(args, "overlay", "src/main/resources/overlay") + //by default, z.json will be picked, and so failed because non-lenient + + //z does not exist + assertThrows { initAndRun(args) } + } + } + + + @Test + fun testRunEM_Overlay_folder_filtered() { + + runTestHandlingFlakyAndCompilation( + "Overlay_folder_filtered", + 100 + ) { args: MutableList -> + + setOption(args, "overlay", "src/main/resources/overlay") + //make sure to not pick-up z.json + setOption(args, "overlayFileSuffixes", ".yaml") + + val solution = initAndRun(args) + + Assertions.assertTrue(solution.individuals.size >= 1) + assertHasAtLeastOne(solution, HttpVerb.GET, 200, "/api/overlay", X) + assertHasAtLeastOne(solution, HttpVerb.GET, 200, "/api/overlay", Y) + } + } + + +} \ No newline at end of file From 8e52082a80ad917883964f5dbac918a165d336a7 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Thu, 12 Mar 2026 15:14:13 +0100 Subject: [PATCH 7/7] overlay test for BB --- .../openapi/v3/overlay/OverlayEMTest.kt | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/overlay/OverlayEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/overlay/OverlayEMTest.kt index 801a9b8e04..91b3ff9910 100644 --- a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/overlay/OverlayEMTest.kt +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/overlay/OverlayEMTest.kt @@ -60,6 +60,28 @@ class OverlayEMTest : SpringTestBase(){ } } + @Test + fun testRunEM_Overlay_Y_BB() { + + runTestHandlingFlakyAndCompilation( + "Overlay_Y_BB", + 100 + ) { args: MutableList -> + + setOption(args, "overlay", "src/main/resources/overlay/subfolder/y.yaml") + setOption(args, "blackBox", "true") + setOption(args, "bbTargetUrl", baseUrlOfSut) + setOption(args, "bbSwaggerUrl", "$baseUrlOfSut/v3/api-docs") + + val solution = initAndRun(args) + + Assertions.assertTrue(solution.individuals.size >= 1) + assertNone(solution, HttpVerb.GET, 200, "/api/overlay", X) + assertHasAtLeastOne(solution, HttpVerb.GET, 200, "/api/overlay", Y) + } + } + + @Test fun testRunEM_Overlay_Z_fail() {