From dce21125264b05ef0c78819b18ff361411f014b2 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Thu, 3 Oct 2024 01:08:30 +0200 Subject: [PATCH 01/10] Create one `Type` object per type reference Fixes #1768 Fixes #1770 --- .../de/fraunhofer/aisec/cpg/TypeManager.kt | 50 ++++++------------- .../fraunhofer/aisec/cpg/graph/TypeBuilder.kt | 38 ++++---------- .../ResolveCallExpressionAmbiguityPass.kt | 21 ++++++++ .../de/fraunhofer/aisec/cpg/test/TestUtils.kt | 5 +- .../frontends/cxx/CXXLanguageFrontendTest.kt | 5 +- .../aisec/cpg_vis_neo4j/Application.kt | 4 +- 6 files changed, 51 insertions(+), 72 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TypeManager.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TypeManager.kt index 3069f74195..9d85fb2e3a 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TypeManager.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TypeManager.kt @@ -28,14 +28,17 @@ package de.fraunhofer.aisec.cpg import de.fraunhofer.aisec.cpg.frontends.CastNotPossible import de.fraunhofer.aisec.cpg.frontends.CastResult import de.fraunhofer.aisec.cpg.frontends.Language +import de.fraunhofer.aisec.cpg.graph.Name import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.TemplateDeclaration +import de.fraunhofer.aisec.cpg.graph.parseName import de.fraunhofer.aisec.cpg.graph.scopes.Scope import de.fraunhofer.aisec.cpg.graph.scopes.TemplateScope import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference import de.fraunhofer.aisec.cpg.graph.types.* import de.fraunhofer.aisec.cpg.passes.Pass import de.fraunhofer.aisec.cpg.passes.ResolveCallExpressionAmbiguityPass +import de.fraunhofer.aisec.cpg.passes.TypeResolver import java.util.* import java.util.concurrent.ConcurrentHashMap import org.slf4j.Logger @@ -57,8 +60,14 @@ class TypeManager { MutableMap> = ConcurrentHashMap() - val firstOrderTypes: MutableSet = ConcurrentHashMap.newKeySet() - val secondOrderTypes: MutableSet = ConcurrentHashMap.newKeySet() + val firstOrderTypes = mutableListOf() + val secondOrderTypes = mutableListOf() + + /** + * A map of declared types by their name. Useful to check for the existence of a declared type + * by its fully qualified name across all scopes. + */ + @PopulatedByPass(TypeResolver::class) val declaredTypes = mutableMapOf() /** * @param recordDeclaration that is instantiated by a template containing parameterizedtypes @@ -200,26 +209,9 @@ class TypeManager { } if (t.isFirstOrderType) { - // Make sure we only ever return one unique object per type - if (!firstOrderTypes.add(t)) { - return firstOrderTypes.first { it == t && it is T } as T - } else { - log.trace( - "Registering unique first order type {}{}", - t.name, - if ((t as? ObjectType)?.generics?.isNotEmpty() == true) { - " with generics ${t.generics.joinToString(",", "[", "]") { it.name.toString() }}" - } else { - "" - } - ) - } + synchronized(firstOrderTypes) { firstOrderTypes.add(t) } } else if (t is SecondOrderType) { - if (!secondOrderTypes.add(t)) { - return secondOrderTypes.first { it == t && it is T } as T - } else { - log.trace("Registering unique second order type {}", t.name) - } + synchronized(secondOrderTypes) { secondOrderTypes.add(t) } } return t @@ -240,25 +232,13 @@ class TypeManager { * This function returns the first (there should be only one) [Type] with the given [fqn] that * is [Type.Origin.RESOLVED]. */ - fun lookupResolvedType( - fqn: CharSequence, - generics: List? = null, - language: Language<*>? = null - ): Type? { + fun lookupResolvedType(fqn: CharSequence, language: Language<*>? = null): Type? { var primitiveType = language?.getSimpleTypeOf(fqn) if (primitiveType != null) { return primitiveType } - return firstOrderTypes.firstOrNull { - (it.typeOrigin == Type.Origin.RESOLVED || it.typeOrigin == Type.Origin.GUESSED) && - it.root.name == fqn && - if (generics != null) { - (it as? ObjectType)?.generics == generics - } else { - true - } - } + return declaredTypes[language.parseName(fqn)] } } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/TypeBuilder.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/TypeBuilder.kt index a0a0fde85a..10f318283f 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/TypeBuilder.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/TypeBuilder.kt @@ -113,34 +113,16 @@ fun LanguageProvider.objectType( "Could not create type: translation context not available" ) - val scope = c.scopeManager.currentScope - - synchronized(c.typeManager.firstOrderTypes) { - // We can try to look up the type by its name and return it, if it already exists. - var type = - c.typeManager.firstOrderTypes.firstOrNull { - it is ObjectType && - it.name == name && - it.scope == scope && - it.generics == generics && - it.language == language - } - if (type != null) { - return type - } - - // Otherwise, we either need to create the type because of the generics or because we do not - // know the type yet. - type = ObjectType(name, generics, false, language) - // Apply our usual metadata, such as scope, code, location, if we have any. Make sure only - // to refer by the local name because we will treat types as sort of references when - // creating them and resolve them later. - type.applyMetadata(this, name, rawNode = rawNode, localNameOnly = true) - - // Piping it through register type will ensure that in any case we return the one unique - // type object (per scope) for it. - return c.typeManager.registerType(type) - } + // Otherwise, we either need to create the type because of the generics or because we do not + // know the type yet. + var type = ObjectType(name, generics, false, language) + // Apply our usual metadata, such as scope, code, location, if we have any. Make sure only + // to refer by the local name because we will treat types as sort of references when + // creating them and resolve them later. + type.applyMetadata(this, name, rawNode = rawNode, localNameOnly = true) + + // Piping it through register type will ensure that we know the type and can resolve it later + return c.typeManager.registerType(type) } /** diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ResolveCallExpressionAmbiguityPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ResolveCallExpressionAmbiguityPass.kt index ecd614701e..d47a73606a 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ResolveCallExpressionAmbiguityPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ResolveCallExpressionAmbiguityPass.kt @@ -116,6 +116,27 @@ class ResolveCallExpressionAmbiguityPass(ctx: TranslationContext) : TranslationU } } + /** This function checks whether our [Reference] refers to a [Type]. */ + private fun lookupPotentialTypeFromReference(ref: Reference): Type? { + var name = ref.name + var scope = ref.scope + + // First, check if it is a simple type + var type = ref.language?.getSimpleTypeOf(name) + if (type != null) { + return type + } + + // This could also be a typedef + type = scopeManager.typedefFor(name, scope) + if (type != null) { + return type + } + + // Lastly, check if the reference contains a symbol that points to type (declaration) + return scopeManager.lookupUniqueTypeSymbolByName(name, scope)?.declaredType + } + override fun cleanup() { // Nothing to do } diff --git a/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/test/TestUtils.kt b/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/test/TestUtils.kt index 04410153f9..34c213b906 100644 --- a/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/test/TestUtils.kt +++ b/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/test/TestUtils.kt @@ -317,8 +317,7 @@ fun assertLiteralValue(expected: T, expr: Expression?, message: Strin assertEquals(expected, assertIs>(expr).value, message) } -fun ContextProvider.assertResolvedType(fqn: String, generics: List? = null): Type { - var type = - ctx?.typeManager?.lookupResolvedType(fqn, generics, (this as? LanguageProvider)?.language) +fun ContextProvider.assertResolvedType(fqn: String): Type { + var type = ctx?.typeManager?.lookupResolvedType(fqn, (this as? LanguageProvider)?.language) return assertNotNull(type) } diff --git a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontendTest.kt b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontendTest.kt index 4e6243fd57..8592f8d0f7 100644 --- a/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontendTest.kt +++ b/cpg-language-cxx/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/cxx/CXXLanguageFrontendTest.kt @@ -61,10 +61,7 @@ internal class CXXLanguageFrontendTest : BaseTest() { val decl = main val ls = decl.variables["ls"] assertNotNull(ls) - assertEquals( - assertResolvedType("std::vector", listOf(assertResolvedType("int"))), - ls.type - ) + assertEquals(assertResolvedType("std::vector"), ls.type) assertLocalName("ls", ls) val forEachStatement = decl.forEachLoops.firstOrNull() diff --git a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt index 1ae5207abd..8afdd6dbdd 100644 --- a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt +++ b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt @@ -520,8 +520,8 @@ class Application : Callable { if (!noDefaultPasses) { translationConfiguration.defaultPasses() - translationConfiguration.registerPass() - translationConfiguration.registerPass() + // translationConfiguration.registerPass() + // translationConfiguration.registerPass() } if (customPasses != "DEFAULT") { val pieces = customPasses.split(",") From 40a1ee6b1de539368729a113f9ee670d45ba2eaa Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Tue, 8 Oct 2024 23:22:33 +0200 Subject: [PATCH 02/10] Partial rollback --- .../de/fraunhofer/aisec/cpg/TypeManager.kt | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TypeManager.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TypeManager.kt index 9d85fb2e3a..95a7f6e9ef 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TypeManager.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TypeManager.kt @@ -31,7 +31,6 @@ import de.fraunhofer.aisec.cpg.frontends.Language import de.fraunhofer.aisec.cpg.graph.Name import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.TemplateDeclaration -import de.fraunhofer.aisec.cpg.graph.parseName import de.fraunhofer.aisec.cpg.graph.scopes.Scope import de.fraunhofer.aisec.cpg.graph.scopes.TemplateScope import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference @@ -232,13 +231,25 @@ class TypeManager { * This function returns the first (there should be only one) [Type] with the given [fqn] that * is [Type.Origin.RESOLVED]. */ - fun lookupResolvedType(fqn: CharSequence, language: Language<*>? = null): Type? { + fun lookupResolvedType( + fqn: CharSequence, + generics: List? = null, + language: Language<*>? = null + ): Type? { var primitiveType = language?.getSimpleTypeOf(fqn) if (primitiveType != null) { return primitiveType } - return declaredTypes[language.parseName(fqn)] + return firstOrderTypes.firstOrNull { + (it.typeOrigin == Type.Origin.RESOLVED || it.typeOrigin == Type.Origin.GUESSED) && + it.root.name == fqn && + if (generics != null) { + (it as? ObjectType)?.generics == generics + } else { + true + } + } } } From 01a96da02a580b4add3255b6a10cb0e8778189cb Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Tue, 8 Oct 2024 23:45:49 +0200 Subject: [PATCH 03/10] ++ --- .../ResolveCallExpressionAmbiguityPass.kt | 21 ---------- .../passes/inference/DFGFunctionSummaries.kt | 39 ++++++++----------- 2 files changed, 16 insertions(+), 44 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ResolveCallExpressionAmbiguityPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ResolveCallExpressionAmbiguityPass.kt index d47a73606a..ecd614701e 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ResolveCallExpressionAmbiguityPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ResolveCallExpressionAmbiguityPass.kt @@ -116,27 +116,6 @@ class ResolveCallExpressionAmbiguityPass(ctx: TranslationContext) : TranslationU } } - /** This function checks whether our [Reference] refers to a [Type]. */ - private fun lookupPotentialTypeFromReference(ref: Reference): Type? { - var name = ref.name - var scope = ref.scope - - // First, check if it is a simple type - var type = ref.language?.getSimpleTypeOf(name) - if (type != null) { - return type - } - - // This could also be a typedef - type = scopeManager.typedefFor(name, scope) - if (type != null) { - return type - } - - // Lastly, check if the reference contains a symbol that points to type (declaration) - return scopeManager.lookupUniqueTypeSymbolByName(name, scope)?.declaredType - } - override fun cleanup() { // Nothing to do } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt index 61e58c87d8..254c92723a 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt @@ -33,11 +33,8 @@ import com.fasterxml.jackson.module.kotlin.registerKotlinModule import de.fraunhofer.aisec.cpg.IncompatibleSignature import de.fraunhofer.aisec.cpg.ancestors import de.fraunhofer.aisec.cpg.frontends.CastNotPossible -import de.fraunhofer.aisec.cpg.graph.ContextProvider -import de.fraunhofer.aisec.cpg.graph.LanguageProvider import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.declarations.* -import de.fraunhofer.aisec.cpg.graph.objectType import de.fraunhofer.aisec.cpg.graph.parseName import de.fraunhofer.aisec.cpg.graph.types.Type import de.fraunhofer.aisec.cpg.graph.unknownType @@ -121,6 +118,7 @@ class DFGFunctionSummaries { val language = functionDecl.language val languageName = language?.javaClass?.name val methodName = functionDecl.name + val typeManager = functionDecl.ctx?.typeManager ?: return null // The language and the method name have to match. If a signature is specified, it also has // to match to the one of the FunctionDeclaration, null indicates that we accept everything. @@ -131,7 +129,12 @@ class DFGFunctionSummaries { // Split the name if we have a FQN val entryMethodName = language.parseName(it.methodName) val entryRecord = - entryMethodName.parent?.let { provider.lookupType(entryMethodName.parent) } + entryMethodName.parent?.let { + typeManager.lookupResolvedType( + entryMethodName.parent, + language = language + ) + } methodName.lastPartsMatch( entryMethodName.localName ) && // The local name has to match @@ -148,7 +151,10 @@ class DFGFunctionSummaries { (it.signature == null || functionDecl.matchesSignature( it.signature.map { signatureType -> - provider.lookupType(signatureType) + typeManager.lookupResolvedType( + signatureType, + language = language + ) ?: functionDecl.unknownType() } ) != IncompatibleSignature) } else { @@ -184,7 +190,7 @@ class DFGFunctionSummaries { .map { Pair( language.parseName(it.methodName).parent?.let { it1 -> - functionDecl.objectType(it1) + typeManager.lookupResolvedType(it1, language = language) }, it ) @@ -215,13 +221,15 @@ class DFGFunctionSummaries { while (mostPreciseClassEntries.size > 1 && argIndex < maxSignature) { mostPreciseType = mostPreciseClassEntries.first().signature?.get(argIndex)?.let { - functionDecl.objectType(it) + typeManager.lookupResolvedType(it, language = language) } superTypes = mostPreciseType?.ancestors?.map { it.type } ?: setOf() val newMostPrecise = mutableListOf() for (entry in mostPreciseClassEntries) { val currentType = - entry.signature?.get(argIndex)?.let { functionDecl.objectType(it) } + entry.signature?.get(argIndex)?.let { + typeManager.lookupResolvedType(it, language = language) + } if (currentType == mostPreciseType) { newMostPrecise.add(entry) } else if (currentType in superTypes) { @@ -356,19 +364,4 @@ class DFGFunctionSummaries { val log: Logger = LoggerFactory.getLogger(DFGFunctionSummaries::class.java) } - - fun ContextProvider.lookupType(fqn: CharSequence): Type { - // Try to look up the type from the specified FQN string - var type = - ctx?.typeManager?.lookupResolvedType( - fqn.toString(), - language = (this as? LanguageProvider)?.language - ) - return if (type == null) { - log.warn("Could not find specified type $fqn. Using UnknownType") - this.unknownType() - } else { - type - } - } } From 907315a5d729aa6b49709b6a4bcb18cf9e20af72 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Wed, 9 Oct 2024 00:08:27 +0200 Subject: [PATCH 04/10] Test still broken --- .../enhancements/DFGFunctionSummariesTest.kt | 76 ++++++++++--------- 1 file changed, 39 insertions(+), 37 deletions(-) diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt index da08592c2f..68158c6137 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/enhancements/DFGFunctionSummariesTest.kt @@ -84,46 +84,48 @@ class DFGFunctionSummariesTest { .build { translationResult { translationUnit("DfgInferredCall.c") { - function("main", t("int")) { - body { - // We need three types with a type hierarchy. - val objectType = t("test.Object") - val listType = t("test.List") - ctx?.let { - val recordDecl = - this@translationUnit.startInference(it) - ?.inferRecordDeclaration( - listType, - ) - listType.recordDeclaration = recordDecl - recordDecl?.addSuperClass(objectType) - listType.superTypes.add(objectType) - } + namespace("test") { + // We need three types with a type hierarchy. + val objectType = t("test.Object") + val listType = t("test.List") + ctx?.let { + val recordDecl = + startInference(it) + ?.inferRecordDeclaration( + listType, + ) + listType.recordDeclaration = recordDecl + recordDecl?.addSuperClass(objectType) + listType.superTypes.add(objectType) + } - val specialListType = t("test.SpecialList") - ctx?.let { - val recordDecl = - this@translationUnit.startInference(it) - ?.inferRecordDeclaration( - specialListType, - ) - specialListType.recordDeclaration = recordDecl - recordDecl?.addSuperClass(listType) - specialListType.superTypes.add(listType) - } + val specialListType = t("test.SpecialList") + ctx?.let { + val recordDecl = + startInference(it) + ?.inferRecordDeclaration( + specialListType, + ) + specialListType.recordDeclaration = recordDecl + recordDecl?.addSuperClass(listType) + specialListType.superTypes.add(listType) + } - val verySpecialListType = t("test.VerySpecialList") - ctx?.let { - val recordDecl = - this@translationUnit.startInference(it) - ?.inferRecordDeclaration( - specialListType, - ) - specialListType.recordDeclaration = recordDecl - recordDecl?.addSuperClass(listType) - specialListType.superTypes.add(listType) - } + val verySpecialListType = t("test.VerySpecialList") + ctx?.let { + val recordDecl = + startInference(it) + ?.inferRecordDeclaration( + verySpecialListType, + ) + verySpecialListType.recordDeclaration = recordDecl + recordDecl?.addSuperClass(specialListType) + verySpecialListType.superTypes.add(listType) + } + } + function("main", t("int")) { + body { memberCall("addAll", construct("test.VerySpecialList")) { literal(1, t("int")) construct("test.Object") From f0976988b00da5452cd6795cd9cf7aebb77a3c2a Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Wed, 9 Oct 2024 00:37:51 +0200 Subject: [PATCH 05/10] Fixed function summaries test --- .../passes/inference/DFGFunctionSummaries.kt | 90 +++++++++---------- 1 file changed, 40 insertions(+), 50 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt index 254c92723a..873aa1d873 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/DFGFunctionSummaries.kt @@ -31,7 +31,7 @@ import com.fasterxml.jackson.dataformat.yaml.YAMLFactory import com.fasterxml.jackson.module.kotlin.readValue import com.fasterxml.jackson.module.kotlin.registerKotlinModule import de.fraunhofer.aisec.cpg.IncompatibleSignature -import de.fraunhofer.aisec.cpg.ancestors +import de.fraunhofer.aisec.cpg.SignatureMatches import de.fraunhofer.aisec.cpg.frontends.CastNotPossible import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.declarations.* @@ -114,7 +114,6 @@ class DFGFunctionSummaries { private fun findFunctionDeclarationEntry(functionDecl: FunctionDeclaration): List? { if (functionToDFGEntryMap.isEmpty()) return null - val provider = functionDecl val language = functionDecl.language val languageName = language?.javaClass?.name val methodName = functionDecl.name @@ -191,58 +190,49 @@ class DFGFunctionSummaries { Pair( language.parseName(it.methodName).parent?.let { it1 -> typeManager.lookupResolvedType(it1, language = language) - }, + } ?: language.unknownType(), it ) } - var mostPreciseClassEntries = mutableListOf() - var mostPreciseType = typeEntryList.first().first - var superTypes = mostPreciseType?.ancestors?.map { it.type } ?: setOf() - for (typeEntry in typeEntryList) { - if (typeEntry.first == mostPreciseType) { - mostPreciseClassEntries.add(typeEntry.second) - } else if (typeEntry.first in superTypes) { - mostPreciseClassEntries.clear() - mostPreciseClassEntries.add(typeEntry.second) - mostPreciseType = typeEntry.first - superTypes = mostPreciseType?.ancestors?.map { it.type } ?: setOf() - } - } - val maxSignature = mostPreciseClassEntries.mapNotNull { it.signature?.size }.max() - if (mostPreciseClassEntries.size > 1) { - mostPreciseClassEntries = - mostPreciseClassEntries - .filter { it.signature?.size == maxSignature } - .toMutableList() - } - // Filter parameter types. We start with parameter 0 and continue. Let's hope we remove - // some entries here. - var argIndex = 0 - while (mostPreciseClassEntries.size > 1 && argIndex < maxSignature) { - mostPreciseType = - mostPreciseClassEntries.first().signature?.get(argIndex)?.let { - typeManager.lookupResolvedType(it, language = language) - } - superTypes = mostPreciseType?.ancestors?.map { it.type } ?: setOf() - val newMostPrecise = mutableListOf() - for (entry in mostPreciseClassEntries) { - val currentType = - entry.signature?.get(argIndex)?.let { - typeManager.lookupResolvedType(it, language = language) - } - if (currentType == mostPreciseType) { - newMostPrecise.add(entry) - } else if (currentType in superTypes) { - newMostPrecise.clear() - newMostPrecise.add(entry) - mostPreciseType = currentType - superTypes = mostPreciseType?.ancestors?.map { it.type } ?: setOf() + val uniqueTypes = typeEntryList.map { it.first }.distinct() + val targetType = + language.parseName(functionDecl.name).parent?.let { it1 -> + typeManager.lookupResolvedType(it1, language = language) + } ?: language.unknownType() + + var mostPreciseType = + uniqueTypes + .map { Pair(it, language?.tryCast(targetType, it)) } + .sortedBy { it.second?.depthDistance } + .firstOrNull() + ?.first + + var mostPreciseClassEntries = + typeEntryList.filter { it.first == mostPreciseType }.map { it.second } + + var signatureResults = + mostPreciseClassEntries + .map { + Pair( + it, + functionDecl.matchesSignature( + it.signature?.map { + typeManager.lookupResolvedType(it, language = language) + ?: language.unknownType() + } ?: listOf() + ) + ) } - } - argIndex++ - mostPreciseClassEntries = newMostPrecise - } - functionToDFGEntryMap[mostPreciseClassEntries.first()] + .filter { it.second is SignatureMatches } + .associate { it } + + val rankings = signatureResults.entries.map { Pair(it.value.ranking, it.key) } + + // Find the best (lowest) rank and find functions with the specific rank + val bestRanking = rankings.minBy { it.first }.first + val list = rankings.filter { it.first == bestRanking }.map { it.second } + + functionToDFGEntryMap[list.first()] } else { null } From adbb7ba288e9aa0bf4d30de16d25a3391fac118f Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Wed, 9 Oct 2024 00:40:07 +0200 Subject: [PATCH 06/10] cleanup --- .../main/kotlin/de/fraunhofer/aisec/cpg/TypeManager.kt | 8 -------- 1 file changed, 8 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TypeManager.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TypeManager.kt index 95a7f6e9ef..60117cc4c3 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TypeManager.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TypeManager.kt @@ -28,7 +28,6 @@ package de.fraunhofer.aisec.cpg import de.fraunhofer.aisec.cpg.frontends.CastNotPossible import de.fraunhofer.aisec.cpg.frontends.CastResult import de.fraunhofer.aisec.cpg.frontends.Language -import de.fraunhofer.aisec.cpg.graph.Name import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.TemplateDeclaration import de.fraunhofer.aisec.cpg.graph.scopes.Scope @@ -37,7 +36,6 @@ import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference import de.fraunhofer.aisec.cpg.graph.types.* import de.fraunhofer.aisec.cpg.passes.Pass import de.fraunhofer.aisec.cpg.passes.ResolveCallExpressionAmbiguityPass -import de.fraunhofer.aisec.cpg.passes.TypeResolver import java.util.* import java.util.concurrent.ConcurrentHashMap import org.slf4j.Logger @@ -62,12 +60,6 @@ class TypeManager { val firstOrderTypes = mutableListOf() val secondOrderTypes = mutableListOf() - /** - * A map of declared types by their name. Useful to check for the existence of a declared type - * by its fully qualified name across all scopes. - */ - @PopulatedByPass(TypeResolver::class) val declaredTypes = mutableMapOf() - /** * @param recordDeclaration that is instantiated by a template containing parameterizedtypes * @param name of the ParameterizedType we want to get From 11a1975223246439f06603607d25e5eacff92122 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Fri, 15 Nov 2024 13:30:23 +0100 Subject: [PATCH 07/10] Fixed crash in Java hiearchy resolver --- .../fraunhofer/aisec/cpg/graph/NodeBuilder.kt | 4 +-- .../de/fraunhofer/aisec/cpg/test/TestUtils.kt | 2 +- .../JavaExternalTypeHierarchyResolver.kt | 12 ++++--- .../aisec/cpg/graph/types/TypeTests.kt | 32 +++++++++++-------- 4 files changed, 30 insertions(+), 20 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/NodeBuilder.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/NodeBuilder.kt index ac060fa20a..5a2e01a263 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/NodeBuilder.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/NodeBuilder.kt @@ -299,8 +299,8 @@ fun T.codeAndLocationFromOtherRawNode(rawNode: AstNode?): T * are between the child nodes. * * @param parentNode Used to extract the code for this node. - * @param newLineType The char(s) used to describe a new line, usually either "\n" or "\r\n". This - * is needed because the location block spanning the children usually comprises more than one + * @param lineBreakSequence The char(s) used to describe a new line, usually either "\n" or "\r\n". + * This is needed because the location block spanning the children usually comprises more than one * line. */ context(CodeAndLocationProvider) diff --git a/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/test/TestUtils.kt b/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/test/TestUtils.kt index 34c213b906..c8f51b80f8 100644 --- a/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/test/TestUtils.kt +++ b/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/test/TestUtils.kt @@ -300,7 +300,7 @@ fun assertUsageOfMemberAndBase(usingNode: Node?, usedBase: Node?, usedMember: De } fun assertFullName(fqn: String, node: Node?, message: String? = null) { - assertNotNull(node) + assertNotNull(node, message) assertEquals(fqn, node.name.toString(), message) } diff --git a/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/JavaExternalTypeHierarchyResolver.kt b/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/JavaExternalTypeHierarchyResolver.kt index 84a75a95b9..8bbf9d2b24 100644 --- a/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/JavaExternalTypeHierarchyResolver.kt +++ b/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/JavaExternalTypeHierarchyResolver.kt @@ -36,6 +36,7 @@ import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.scopes.Scope import de.fraunhofer.aisec.cpg.graph.types.Type import de.fraunhofer.aisec.cpg.helpers.CommonPath +import de.fraunhofer.aisec.cpg.passes.SymbolResolver.Companion.LOGGER import de.fraunhofer.aisec.cpg.passes.configuration.DependsOn import de.fraunhofer.aisec.cpg.passes.configuration.ExecuteBefore import de.fraunhofer.aisec.cpg.passes.configuration.RequiredFrontend @@ -71,10 +72,12 @@ class JavaExternalTypeHierarchyResolver(ctx: TranslationContext) : ComponentPass } // Iterate over all known types and add their (direct) supertypes. - for (t in typeManager.firstOrderTypes) { + var types = typeManager.firstOrderTypes.toList() + for (t in types) { val symbol = resolver.tryToSolveType(t.typeName) if (symbol.isSolved) { try { + val resolvedSuperTypes = symbol.correspondingDeclaration.getAncestors(true) for (anc in resolvedSuperTypes) { // We need to try to resolve the type first in order to create weirdly @@ -90,10 +93,11 @@ class JavaExternalTypeHierarchyResolver(ctx: TranslationContext) : ComponentPass // Add all resolved supertypes to the type. t.superTypes.add(superType) } - } catch (e: UnsolvedSymbolException) { - // Even if the symbol itself is resolved, "getAncestors()" may throw exception. + } catch (_: UnsolvedSymbolException) { + // Even if the symbol itself is resolved, "getAncestors()" may throw + // exception. LOGGER.warn( - "Could not resolve supertypes of ${symbol.correspondingDeclaration}" + "Could not resolve supertypes of ${symbol?.correspondingDeclaration}" ) } } diff --git a/cpg-language-java/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/types/TypeTests.kt b/cpg-language-java/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/types/TypeTests.kt index 5b608b175b..07b0c56130 100644 --- a/cpg-language-java/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/types/TypeTests.kt +++ b/cpg-language-java/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/types/TypeTests.kt @@ -111,18 +111,20 @@ internal class TypeTests : BaseTest() { fun testCommonTypeTestJava() { val topLevel = Path.of("src", "test", "resources", "compiling", "hierarchy") val result = analyze("java", topLevel, true) { it.registerLanguage(JavaLanguage()) } - val root = assertNotNull(result.records["multistep.Root"]).toType() - val level0 = assertNotNull(result.records["multistep.Level0"]).toType() - val level1 = assertNotNull(result.records["multistep.Level1"]).toType() - val level1b = assertNotNull(result.records["multistep.Level1B"]).toType() - val level2 = assertNotNull(result.records["multistep.Level2"]).toType() - val unrelated = assertNotNull(result.records["multistep.Unrelated"]).toType() - println( - result.finalCtx.typeManager.firstOrderTypes - .filter { it.typeName == "multistep.Root" } - .map { it.superTypes } - ) - getCommonTypeTestGeneral(root, level0, level1, level1b, level2, unrelated) + with(result) { + val root = assertResolvedType("multistep.Root") + val level0 = assertResolvedType("multistep.Level0") + val level1 = assertResolvedType("multistep.Level1") + val level1b = assertResolvedType("multistep.Level1B") + val level2 = assertResolvedType("multistep.Level2") + val unrelated = assertResolvedType("multistep.Unrelated") + println( + result.finalCtx.typeManager.firstOrderTypes + .filter { it.typeName == "multistep.Root" } + .map { it.superTypes } + ) + getCommonTypeTestGeneral(root, level0, level1, level1b, level2, unrelated) + } } private fun getCommonTypeTestGeneral( @@ -169,7 +171,11 @@ internal class TypeTests : BaseTest() { // Check unrelated type behavior: No common root class for (t in listOf(root, level0, level1, level1b, level2)) { - assertFullName("java.lang.Object", setOf(unrelated, t).commonType) + assertFullName( + "java.lang.Object", + setOf(unrelated, t).commonType, + "${t.typeName} and ${unrelated.typeName} do not have a common type (java.lang.Object) which they should" + ) } } } From b46d5318232304a9c99b97517cfb9496cf0a19d1 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Thu, 28 Nov 2024 22:33:38 +0100 Subject: [PATCH 08/10] Workaround for test coverage bug --- .github/workflows/build.yml | 10 +++++++++- .../de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt | 4 ++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 70030ee32a..785475de5c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -68,11 +68,19 @@ jobs: id: build env: VERSION: ${{ env.version }} + - name: Prepare report.xml for Codecov + run: | + # this is needed because codecov incorrectly reports lines that have no coverage information (good or bad) as a miss + # See https://github.com/codecov/feedback/issues/564 and https://github.com/Kotlin/kotlinx-kover/issues/699. + # Actually these lines should just not exist in the coverage XML file, since they are only structural elements, such + # as brackets. + cat cpg-all/build/reports/kover/report.xml | grep -v 'mi="0" ci="0" mb="0" cb="0"' > cpg-all/build/reports/kover/report-codecov.xml + rm cpg-all/build/reports/kover/report.xml - name: Upload Code Coverage uses: codecov/codecov-action@v5 with: fail_ci_if_error: true - files: ./cpg-all/build/reports/kover/report.xml + files: ./cpg-all/build/reports/kover/report-codecov.xml token: ${{ secrets.CODECOV_TOKEN }} verbose: true - name: Prepare test and coverage reports diff --git a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt index 8afdd6dbdd..1ae5207abd 100644 --- a/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt +++ b/cpg-neo4j/src/main/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Application.kt @@ -520,8 +520,8 @@ class Application : Callable { if (!noDefaultPasses) { translationConfiguration.defaultPasses() - // translationConfiguration.registerPass() - // translationConfiguration.registerPass() + translationConfiguration.registerPass() + translationConfiguration.registerPass() } if (customPasses != "DEFAULT") { val pieces = customPasses.split(",") From 6d40ad3e8eeb0fc8264d37584d32b9cb6ab96389 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Thu, 28 Nov 2024 22:47:35 +0100 Subject: [PATCH 09/10] Fix --- .../kotlin/de/fraunhofer/aisec/cpg/test/TestUtils.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/test/TestUtils.kt b/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/test/TestUtils.kt index c8f51b80f8..21435dddbf 100644 --- a/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/test/TestUtils.kt +++ b/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/test/TestUtils.kt @@ -318,6 +318,7 @@ fun assertLiteralValue(expected: T, expr: Expression?, message: Strin } fun ContextProvider.assertResolvedType(fqn: String): Type { - var type = ctx?.typeManager?.lookupResolvedType(fqn, (this as? LanguageProvider)?.language) + var type = + ctx?.typeManager?.lookupResolvedType(fqn, language = (this as? LanguageProvider)?.language) return assertNotNull(type) } From af6be91c0ca8adafb71ce42415fa0d8ba98b90ef Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Fri, 29 Nov 2024 22:49:02 +0100 Subject: [PATCH 10/10] Removing unncessary exception handling --- .../JavaExternalTypeHierarchyResolver.kt | 35 +++++++------------ 1 file changed, 12 insertions(+), 23 deletions(-) diff --git a/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/JavaExternalTypeHierarchyResolver.kt b/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/JavaExternalTypeHierarchyResolver.kt index 8bbf9d2b24..1bd43e1d1c 100644 --- a/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/JavaExternalTypeHierarchyResolver.kt +++ b/cpg-language-java/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/JavaExternalTypeHierarchyResolver.kt @@ -25,7 +25,6 @@ */ package de.fraunhofer.aisec.cpg.passes -import com.github.javaparser.resolution.UnsolvedSymbolException import com.github.javaparser.symbolsolver.resolution.typesolvers.CombinedTypeSolver import com.github.javaparser.symbolsolver.resolution.typesolvers.JavaParserTypeSolver import com.github.javaparser.symbolsolver.resolution.typesolvers.ReflectionTypeSolver @@ -36,7 +35,6 @@ import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.scopes.Scope import de.fraunhofer.aisec.cpg.graph.types.Type import de.fraunhofer.aisec.cpg.helpers.CommonPath -import de.fraunhofer.aisec.cpg.passes.SymbolResolver.Companion.LOGGER import de.fraunhofer.aisec.cpg.passes.configuration.DependsOn import de.fraunhofer.aisec.cpg.passes.configuration.ExecuteBefore import de.fraunhofer.aisec.cpg.passes.configuration.RequiredFrontend @@ -76,29 +74,20 @@ class JavaExternalTypeHierarchyResolver(ctx: TranslationContext) : ComponentPass for (t in types) { val symbol = resolver.tryToSolveType(t.typeName) if (symbol.isSolved) { - try { + val resolvedSuperTypes = symbol.correspondingDeclaration.getAncestors(true) + for (anc in resolvedSuperTypes) { + // We need to try to resolve the type first in order to create weirdly + // scoped types + var superType = typeManager.lookupResolvedType(anc.qualifiedName) - val resolvedSuperTypes = symbol.correspondingDeclaration.getAncestors(true) - for (anc in resolvedSuperTypes) { - // We need to try to resolve the type first in order to create weirdly - // scoped types - var superType = typeManager.lookupResolvedType(anc.qualifiedName) - - // Otherwise, we can create this in the global scope - if (superType == null) { - superType = provider.objectType(anc.qualifiedName) - superType.typeOrigin = Type.Origin.RESOLVED - } - - // Add all resolved supertypes to the type. - t.superTypes.add(superType) + // Otherwise, we can create this in the global scope + if (superType == null) { + superType = provider.objectType(anc.qualifiedName) + superType.typeOrigin = Type.Origin.RESOLVED } - } catch (_: UnsolvedSymbolException) { - // Even if the symbol itself is resolved, "getAncestors()" may throw - // exception. - LOGGER.warn( - "Could not resolve supertypes of ${symbol?.correspondingDeclaration}" - ) + + // Add all resolved supertypes to the type. + t.superTypes.add(superType) } } }