Greybox fuzzer

settings.gradle.kts

@@ -23,6 +23,7 @@ include("utbot-intellij")
 include("utbot-sample")
 include("utbot-fuzzers")
 include("utbot-fuzzing")
+include("utbot-greyboxfuzzer")
 include("utbot-junit-contest")
 include("utbot-analytics")
 include("utbot-analytics-torch")

utbot-cli/src/main/kotlin/org/utbot/cli/GenerateTestsAbstractCommand.kt

@@ -144,8 +144,21 @@ abstract class GenerateTestsAbstractCommand(name: String, help: String) :
     protected fun getWorkingDirectory(classFqn: String): Path? {
         val classRelativePath = classFqnToPath(classFqn) + ".class"
         val classAbsoluteURL = classLoader.getResource(classRelativePath) ?: return null
-        val classAbsolutePath = replaceSeparator(classAbsoluteURL.toPath().toString())
-            .removeSuffix(classRelativePath)
+        val classAbsolutePath =
+            if (UtSettings.useGreyBoxFuzzing) {
+                if (classAbsoluteURL.toURI().scheme == "jar") {
+                    replaceSeparator(classAbsoluteURL.file.removePrefix("file:"))
+                        .removeSuffix(classRelativePath)
+                        .removeSuffix("/")
+                        .removeSuffix("!")
+                } else {
+                    replaceSeparator(classAbsoluteURL.toPath().toString())
+                        .removeSuffix(classRelativePath)
+                }
+            } else {
+                replaceSeparator(classAbsoluteURL.toPath().toString())
+                    .removeSuffix(classRelativePath)
+            }
         return Paths.get(classAbsolutePath)
     }
 

utbot-core/src/main/kotlin/org/utbot/common/FileUtil.kt

@@ -90,7 +90,10 @@ object FileUtil {
 
         for (clazz in classes) {
             val path = clazz.toClassFilePath()
-            val resource = clazz.classLoader.getResource(path) ?: error("No such file: $path")
+            val resource =
+                clazz.classLoader.getResource(path)
+                    ?: ClassLoader.getSystemClassLoader().getResource(path)
+                    ?: error("No such file: $path")
 
             if (resource.toURI().scheme == "jar") {
                 val jarLocation = resource.toURI().extractJarName()

utbot-framework-api/src/main/kotlin/org/utbot/framework/UtSettings.kt

@@ -242,6 +242,21 @@ object UtSettings : AbstractSettings(logger, defaultKeyForSettingsPath, defaultS
      */
     var useFuzzing: Boolean by getBooleanProperty(true)
 
+    /**
+     * Set to true to use grey-box fuzzing
+     */
+    var useGreyBoxFuzzing: Boolean by getBooleanProperty(false)
+
+    /**
+     * Set to true to use UtCompositeModels in grey-box fuzzing process
+     */
+    var useCompositeModelsInGreyBoxFuzzing: Boolean by getBooleanProperty(false)
+
+    /**
+     * Set to true to use grey-box fuzzing in competition mode (without asserts generation)
+     */
+    var greyBoxFuzzingCompetitionMode: Boolean by getBooleanProperty(false)
+
     /**
      * Set the total attempts to improve coverage by fuzzer.
      */

utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/UtExecutionResult.kt

@@ -1,5 +1,6 @@
 package org.utbot.framework.plugin.api
 
+import org.utbot.framework.plugin.api.util.objectClassId
 import org.utbot.framework.plugin.api.visible.UtStreamConsumingException
 import java.io.File
 import java.util.LinkedList
@@ -69,6 +70,9 @@ class ConcreteExecutionFailureException(cause: Throwable, errorFile: File, val p
             appendLine("Cause:\n${cause.message}")
             appendLine("Last 1000 lines of the error log ${errorFile.absolutePath}:")
             appendLine("----------------------------------------")
+            if (!errorFile.exists()) {
+                errorFile.createNewFile()
+            }
             errorFile.useLines { lines ->
                 val lastLines = LinkedList<String>()
                 for (line in lines) {
@@ -102,6 +106,11 @@ inline fun UtExecutionResult.onFailure(action: (exception: Throwable) -> Unit):
     return this
 }
 
+fun UtExecutionResult.getOrThrow(): UtModel = when (this) {
+    is UtExecutionSuccess -> model
+    is UtExecutionFailure -> throw exception
+}
+
 fun UtExecutionResult.exceptionOrNull(): Throwable? = when (this) {
     is UtExecutionFailure -> rootCauseException
     is UtExecutionSuccess -> null

utbot-framework/build.gradle

@@ -15,6 +15,7 @@ dependencies {
     api project(':utbot-summary')
     api project(':utbot-framework-api')
     api project(':utbot-rd')
+    api project(':utbot-greyboxfuzzer')
 
     implementation group: 'com.jetbrains.rd', name: 'rd-framework', version: rdVersion
     implementation group: 'com.jetbrains.rd', name: 'rd-core', version: rdVersion

utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt

@@ -32,6 +32,8 @@ import org.utbot.framework.UtSettings.pathSelectorStepsLimit
 import org.utbot.framework.UtSettings.pathSelectorType
 import org.utbot.framework.UtSettings.processUnknownStatesDuringConcreteExecution
 import org.utbot.framework.UtSettings.useDebugVisualization
+import org.utbot.framework.concrete.FuzzerConcreteExecutor
+import org.utbot.framework.concrete.UtFuzzingExecutionInstrumentation
 import org.utbot.framework.util.convertToAssemble
 import org.utbot.framework.plugin.api.*
 import org.utbot.framework.plugin.api.Step
@@ -41,13 +43,18 @@ import org.utbot.framework.util.sootMethod
 import org.utbot.fuzzer.*
 import org.utbot.fuzzing.*
 import org.utbot.fuzzing.utils.Trie
+import org.utbot.greyboxfuzzer.GreyBoxFuzzer
+import org.utbot.greyboxfuzzer.util.FuzzerUtModelConstructor
 import org.utbot.instrumentation.ConcreteExecutor
 import org.utbot.instrumentation.instrumentation.execution.UtConcreteExecutionData
 import org.utbot.instrumentation.instrumentation.execution.UtConcreteExecutionResult
 import org.utbot.instrumentation.instrumentation.execution.UtExecutionInstrumentation
+import org.utbot.instrumentation.instrumentation.execution.constructors.UtModelConstructor
+import org.utbot.instrumentation.instrumentation.execution.phases.ValueConstructionContext
 import soot.jimple.Stmt
 import soot.tagkit.ParamNamesTag
 import java.lang.reflect.Method
+import java.util.*
 import kotlin.system.measureTimeMillis
 
 val logger = KotlinLogging.logger {}
@@ -334,7 +341,7 @@ class UtBotSymbolicEngine(
     fun fuzzing(until: Long = Long.MAX_VALUE, transform: (JavaValueProvider) -> JavaValueProvider = { it }) = flow {
         val isFuzzable = methodUnderTest.parameters.all { classId ->
             classId != Method::class.java.id && // causes the instrumented process crash at invocation
-                classId != Class::class.java.id  // causes java.lang.IllegalAccessException: java.lang.Class at sun.misc.Unsafe.allocateInstance(Native Method)
+                    classId != Class::class.java.id  // causes java.lang.IllegalAccessException: java.lang.Class at sun.misc.Unsafe.allocateInstance(Native Method)
         }
         val hasMethodUnderTestParametersToFuzz = methodUnderTest.parameters.isNotEmpty()
         if (!isFuzzable || !hasMethodUnderTestParametersToFuzz && methodUnderTest.isStatic) {
@@ -416,6 +423,42 @@ class UtBotSymbolicEngine(
         }
     }
 
+    //Simple fuzzing
+    fun greyBoxFuzzing(timeBudget: Long = Long.MAX_VALUE) =
+        flow {
+            val isFuzzable = methodUnderTest.parameters.all { classId ->
+                classId != Method::class.java.id // causes the child process crash at invocation
+            }
+            if (!isFuzzable) {
+                return@flow
+            }
+            val utModelConstructor = UtModelConstructor(IdentityHashMap())
+            val fuzzerUtModelConstructor = FuzzerUtModelConstructor(
+                utModelConstructor::construct,
+                utModelConstructor::computeUnusedIdAndUpdate
+            )
+
+            try {
+                emitAll(
+                    GreyBoxFuzzer(
+                        methodUnderTest,
+                        collectConstantsForGreyBoxFuzzer(methodUnderTest.sootMethod, utModelConstructor),
+                        fuzzerUtModelConstructor,
+                        FuzzerConcreteExecutor(
+                            concreteExecutor.pathsToUserClasses
+                        )::execute,
+                        ValueConstructionContext(UtFuzzingExecutionInstrumentation.instrumentationContext, true)::constructParameters,
+                        timeBudget
+                    ).fuzz()
+                )
+            } catch (e: CancellationException) {
+                logger.debug { "Cancelled by timeout" }
+            } catch (e: Throwable) {
+                emit(UtError("Unexpected fuzzing crash\n${e.stackTraceToString()}", e))
+            }
+            return@flow
+        }
+
     private suspend fun FlowCollector<UtResult>.emitFailedConcreteExecutionResult(
         stateBefore: EnvironmentModels,
         e: ConcreteExecutionFailureException
@@ -556,7 +599,7 @@ private fun ResolvedModels.constructStateForMethod(methodUnderTest: ExecutableId
     return EnvironmentModels(thisInstanceBefore, paramsBefore, statics)
 }
 
-private suspend fun ConcreteExecutor<UtConcreteExecutionResult, UtExecutionInstrumentation>.executeConcretely(
+internal suspend fun ConcreteExecutor<UtConcreteExecutionResult, UtExecutionInstrumentation>.executeConcretely(
     methodUnderTest: ExecutableId,
     stateBefore: EnvironmentModels,
     instrumentation: List<UtInstrumentation>

utbot-framework/src/main/kotlin/org/utbot/external/api/UtBotJavaApi.kt

@@ -70,7 +70,7 @@ object UtBotJavaApi {
 
         val concreteExecutor = ConcreteExecutor(
             UtExecutionInstrumentation,
-            classpath,
+            classpath
         )
 
         testSets.addAll(generateUnitTests(concreteExecutor, methodsForGeneration, classUnderTest))
@@ -145,6 +145,7 @@ object UtBotJavaApi {
         dependencyClassPath: String,
         mockStrategyApi: MockStrategyApi = MockStrategyApi.OTHER_PACKAGES,
         generationTimeoutInMillis: Long = UtSettings.utBotGenerationTimeoutInMillis,
+        isGreyBoxFuzzing: Boolean = false,
         primitiveValuesSupplier: CustomFuzzerValueSupplier = CustomFuzzerValueSupplier { null }
     ): MutableList<UtMethodTestSet> {
         fun createPrimitiveModels(supplier: CustomFuzzerValueSupplier, classId: ClassId): Sequence<UtPrimitiveModel> =
@@ -182,8 +183,12 @@ object UtBotJavaApi {
                     chosenClassesToMockAlways = emptySet(),
                     generationTimeoutInMillis,
                     generate = { symbolicEngine ->
-                        symbolicEngine.fuzzing { defaultModelProvider ->
-                            customModelProvider.withFallback(defaultModelProvider)
+                        if (isGreyBoxFuzzing) {
+                            symbolicEngine.greyBoxFuzzing(generationTimeoutInMillis)
+                        } else {
+                            symbolicEngine.fuzzing { defaultModelProvider ->
+                                customModelProvider.withFallback(defaultModelProvider)
+                            }
                         }
                     }
                 )

utbot-framework/src/main/kotlin/org/utbot/framework/codegen/domain/models/CgElement.kt

@@ -45,6 +45,7 @@ interface CgElement {
             is CgAuxiliaryClass -> visit(element)
             is CgUtilMethod -> visit(element)
             is CgTestMethod -> visit(element)
+            is CgMockMethod -> visit(element)
             is CgErrorTestMethod -> visit(element)
             is CgParameterizedTestDataProviderMethod -> visit(element)
             is CgCommentedAnnotation -> visit(element)
@@ -282,6 +283,17 @@ class CgTestMethod(
     override val requiredFields: List<CgParameterDeclaration> = emptyList(),
 ) : CgMethod(false)
 
+class CgMockMethod(
+    override val name: String,
+    override val returnType: ClassId,
+    override val parameters: List<CgParameterDeclaration>,
+    override val statements: List<CgStatement>,
+    override val exceptions: Set<ClassId>,
+    override val annotations: List<CgAnnotation>,
+    override val documentation: CgDocumentationComment = CgDocumentationComment(emptyList()),
+    override val requiredFields: List<CgParameterDeclaration> = emptyList(),
+) : CgMethod(false)
+
 class CgErrorTestMethod(
     override val name: String,
     override val statements: List<CgStatement>,

utbot-framework/src/main/kotlin/org/utbot/framework/codegen/renderer/CgAbstractRenderer.kt

@@ -74,6 +74,7 @@ import org.utbot.framework.codegen.domain.models.CgStaticFieldAccess
 import org.utbot.framework.codegen.domain.models.CgStaticRunnable
 import org.utbot.framework.codegen.domain.models.CgStaticsRegion
 import org.utbot.framework.codegen.domain.models.CgTestMethod
+import org.utbot.framework.codegen.domain.models.CgMockMethod
 import org.utbot.framework.codegen.domain.models.CgTestMethodCluster
 import org.utbot.framework.codegen.domain.models.CgThisInstance
 import org.utbot.framework.codegen.domain.models.CgThrowStatement
@@ -97,6 +98,7 @@ import org.utbot.framework.plugin.api.util.isArray
 import org.utbot.framework.plugin.api.util.isRefType
 import org.utbot.framework.plugin.api.util.longClassId
 import org.utbot.framework.plugin.api.util.shortClassId
+import org.utbot.framework.plugin.api.util.isPublic
 
 abstract class CgAbstractRenderer(
     val context: CgRendererContext,
@@ -245,6 +247,15 @@ abstract class CgAbstractRenderer(
         visit(element as CgMethod)
     }
 
+    override fun visit(element: CgMockMethod) {
+        renderMethodDocumentation(element)
+        for (annotation in element.annotations) {
+            annotation.accept(this)
+        }
+        renderMethodSignature(element)
+        visit(element as CgMethod)
+    }
+
     override fun visit(element: CgErrorTestMethod) {
         renderMethodDocumentation(element)
         renderMethodSignature(element)
@@ -749,6 +760,7 @@ abstract class CgAbstractRenderer(
     protected val maxParametersAmountInOneLine = 3
 
     protected abstract fun renderMethodSignature(element: CgTestMethod)
+    protected abstract fun renderMethodSignature(element: CgMockMethod)
     protected abstract fun renderMethodSignature(element: CgErrorTestMethod)
     protected abstract fun renderMethodSignature(element: CgParameterizedTestDataProviderMethod)
 

utbot-framework/src/main/kotlin/org/utbot/framework/codegen/renderer/CgJavaRenderer.kt

@@ -35,16 +35,13 @@ import org.utbot.framework.codegen.domain.models.CgClassBody
 import org.utbot.framework.codegen.domain.models.CgFormattedString
 import org.utbot.framework.codegen.domain.models.CgLiteral
 import org.utbot.framework.codegen.domain.models.CgTestMethod
+import org.utbot.framework.codegen.domain.models.CgMockMethod
 import org.utbot.framework.codegen.domain.models.CgTypeCast
 import org.utbot.framework.codegen.domain.models.CgVariable
 import org.utbot.framework.codegen.util.nullLiteral
 import org.utbot.framework.plugin.api.ClassId
 import org.utbot.framework.plugin.api.TypeParameters
-import org.utbot.framework.plugin.api.util.isFinal
-import org.utbot.framework.plugin.api.util.isPrivate
-import org.utbot.framework.plugin.api.util.isProtected
-import org.utbot.framework.plugin.api.util.isPublic
-import org.utbot.framework.plugin.api.util.wrapperByPrimitive
+import org.utbot.framework.plugin.api.util.*
 
 internal class CgJavaRenderer(context: CgRendererContext, printer: CgPrinter = CgPrinterImpl()) :
     CgAbstractRenderer(context, printer) {
@@ -241,6 +238,20 @@ internal class CgJavaRenderer(context: CgRendererContext, printer: CgPrinter = C
         renderExceptions(element)
     }
 
+
+    override fun renderMethodSignature(element: CgMockMethod) {
+        val returnType = element.returnType.asString()
+        print("public $returnType ")
+        print(element.name)
+
+        print("(")
+        val newLinesNeeded = element.parameters.size > maxParametersAmountInOneLine
+        element.parameters.renderSeparated(newLinesNeeded)
+        print(")")
+
+        renderExceptions(element)
+    }
+
     override fun renderMethodSignature(element: CgErrorTestMethod) {
         // error test methods always have void return type
         println("public void ${element.name}()")

utbot-framework/src/main/kotlin/org/utbot/framework/codegen/renderer/CgKotlinRenderer.kt

@@ -41,6 +41,7 @@ import org.utbot.framework.codegen.domain.models.CgClassBody
 import org.utbot.framework.codegen.domain.models.CgFormattedString
 import org.utbot.framework.codegen.domain.models.CgLiteral
 import org.utbot.framework.codegen.domain.models.CgTestMethod
+import org.utbot.framework.codegen.domain.models.CgMockMethod
 import org.utbot.framework.codegen.domain.models.CgTypeCast
 import org.utbot.framework.codegen.domain.models.CgValue
 import org.utbot.framework.codegen.domain.models.CgVariable
@@ -403,6 +404,16 @@ internal class CgKotlinRenderer(context: CgRendererContext, printer: CgPrinter =
         renderMethodReturnType(element)
     }
 
+    override fun renderMethodSignature(element: CgMockMethod) {
+        print("fun ")
+        print(element.name)
+        print("(")
+        val newLines = element.parameters.size > maxParametersAmountInOneLine
+        element.parameters.renderSeparated(newLines)
+        print(")")
+        renderMethodReturnType(element)
+    }
+
     override fun renderMethodSignature(element: CgErrorTestMethod) {
         // error test methods always have void return type
         print("fun ")

utbot-framework/src/main/kotlin/org/utbot/framework/codegen/renderer/CgVisitor.kt

@@ -79,6 +79,7 @@ import org.utbot.framework.codegen.domain.models.CgDocRegularLineStmt
 import org.utbot.framework.codegen.domain.models.CgFormattedString
 import org.utbot.framework.codegen.domain.models.CgNestedClassesRegion
 import org.utbot.framework.codegen.domain.models.CgTestMethod
+import org.utbot.framework.codegen.domain.models.CgMockMethod
 import org.utbot.framework.codegen.domain.models.CgTestMethodCluster
 import org.utbot.framework.codegen.domain.models.CgThisInstance
 import org.utbot.framework.codegen.domain.models.CgThrowStatement
@@ -111,6 +112,7 @@ interface CgVisitor<R> {
     // Methods
     fun visit(element: CgMethod): R
     fun visit(element: CgTestMethod): R
+    fun visit(element: CgMockMethod): R
     fun visit(element: CgErrorTestMethod): R
     fun visit(element: CgParameterizedTestDataProviderMethod): R