From 3c6411d063e6d0740e6c42efa8d5e26ae6a043c3 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Wed, 27 Mar 2024 09:55:01 +0100 Subject: [PATCH] Fixing record inference and making `SymbolResolver` deterministic again (#1476) --- .../de/fraunhofer/aisec/cpg/ScopeManager.kt | 2 +- .../fraunhofer/aisec/cpg/graph/Extensions.kt | 5 +- .../aisec/cpg/passes/SymbolResolver.kt | 4 +- .../aisec/cpg/passes/inference/Inference.kt | 11 ++-- .../golang/GoLanguageFrontendTest.kt | 24 ++++++++ .../src/test/resources/golang/inference.go | 8 +++ .../aisec/cpg_vis_neo4j/ApplicationTest.kt | 57 +++++------------- .../aisec/cpg_vis_neo4j/Neo4JTest.kt | 59 +++++++++++++++++++ cpg-neo4j/src/test/resources/log4j2.xml | 15 +++++ 9 files changed, 131 insertions(+), 54 deletions(-) create mode 100644 cpg-language-go/src/test/resources/golang/inference.go create mode 100644 cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Neo4JTest.kt create mode 100644 cpg-neo4j/src/test/resources/log4j2.xml diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt index 141d1df7a3..d3ca2f15bd 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt @@ -727,7 +727,7 @@ class ScopeManager : ScopeProvider { val scopes = filterScopes { (it is NameScope && it.name == scopeName) } s = if (scopes.isEmpty()) { - Util.errorWithFileLocation( + Util.warnWithFileLocation( node, LOGGER, "Could not find the scope $scopeName needed to resolve the call ${node.name}" diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Extensions.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Extensions.kt index 300e218a9d..996897b198 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Extensions.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Extensions.kt @@ -36,7 +36,6 @@ import de.fraunhofer.aisec.cpg.graph.statements.SwitchStatement import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker -import de.fraunhofer.aisec.cpg.helpers.toIdentitySet import de.fraunhofer.aisec.cpg.passes.EvaluationOrderGraphPass import de.fraunhofer.aisec.cpg.passes.astParent @@ -67,9 +66,9 @@ inline fun Node?.allChildren(noinline predicate: ((T) -> Boolean)? = * include retrieving it from either an individual [TranslationUnitDeclaration] or the complete * [TranslationResult]. */ -val Node.allEOGStarters: Set +val Node.allEOGStarters: List get() { - return this.allChildren().flatMap { it.eogStarters }.toIdentitySet() + return this.allChildren().flatMap { it.eogStarters }.distinct() } @JvmName("astNodes") diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt index aa30db286f..3edf2cb4d7 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt @@ -100,7 +100,7 @@ open class SymbolResolver(ctx: TranslationContext) : ComponentPass(ctx) { for (tu in component.translationUnits) { currentTU = tu // gather all resolution start holders and their start nodes - val nodes = tu.allEOGStarters.toSet() + val nodes = tu.allEOGStarters for (node in nodes) { walker.iterate(node) @@ -700,7 +700,7 @@ open class SymbolResolver(ctx: TranslationContext) : ComponentPass(ctx) { if (root != null && record == null) { record = it.startInference(ctx) - ?.inferRecordDeclaration(it, currentTU, locationHint = call) + ?.inferRecordDeclaration(root, currentTU, locationHint = call) // update the record declaration root.recordDeclaration = record } diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/Inference.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/Inference.kt index 726df0c549..83431ada13 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/Inference.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/inference/Inference.kt @@ -375,11 +375,6 @@ class Inference internal constructor(val start: Node, override val ctx: Translat ) return null } - debugWithFileLocation( - locationHint, - log, - "Encountered an unknown record type ${type.typeName} during a call. We are going to infer that record" - ) return inferInScopeOf(currentTU) { // This could be a class or a struct. We start with a class and may have to fine-tune @@ -387,6 +382,12 @@ class Inference internal constructor(val start: Node, override val ctx: Translat val declaration = newRecordDeclaration(type.typeName, kind) declaration.isInferred = true + debugWithFileLocation( + locationHint, + log, + "Inferred a new record declaration ${declaration.name} (${declaration.kind})" + ) + // Update the type type.recordDeclaration = declaration diff --git a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontendTest.kt b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontendTest.kt index 8a52c81b95..26c6625442 100644 --- a/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontendTest.kt +++ b/cpg-language-go/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/golang/GoLanguageFrontendTest.kt @@ -510,6 +510,30 @@ class GoLanguageFrontendTest : BaseTest() { assertInvokes(printfCall, printf) } + @Test + fun testPointerTypeInference() { + val topLevel = Path.of("src", "test", "resources", "golang") + val result = + analyze( + listOf( + topLevel.resolve("inference.go").toFile(), + ), + topLevel, + true + ) { + it.registerLanguage() + } + assertNotNull(result) + + // There should be only a single one inferred method with that name + val queryDecl = result.methods("Query").singleOrNull() + assertNotNull(queryDecl) + assertTrue(queryDecl.isInferred) + + val query = result.mcalls["Query"] + assertInvokes(query, queryDecl) + } + @Test fun testQualifiedCallInMethod() { val stdLib = Path.of("src", "test", "resources", "golang-std") diff --git a/cpg-language-go/src/test/resources/golang/inference.go b/cpg-language-go/src/test/resources/golang/inference.go new file mode 100644 index 0000000000..bd0b74d842 --- /dev/null +++ b/cpg-language-go/src/test/resources/golang/inference.go @@ -0,0 +1,8 @@ +package p + +import "database/sql" + +func doDB() { + var db *sql.DB + db.Query("SELECT * FROM table") +} diff --git a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt index 412906fbfb..b032b877e9 100644 --- a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt +++ b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/ApplicationTest.kt @@ -26,12 +26,11 @@ package de.fraunhofer.aisec.cpg_vis_neo4j import com.fasterxml.jackson.databind.ObjectMapper -import de.fraunhofer.aisec.cpg.* -import de.fraunhofer.aisec.cpg.graph.builder.* +import de.fraunhofer.aisec.cpg.TranslationManager +import de.fraunhofer.aisec.cpg.TranslationResult import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import de.fraunhofer.aisec.cpg.graph.functions import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression -import de.fraunhofer.aisec.cpg.graph.types.* import java.io.File import java.nio.file.Files import java.nio.file.Paths @@ -41,56 +40,28 @@ import kotlin.reflect.jvm.javaField import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertNotNull -import org.junit.jupiter.api.Tag import org.neo4j.ogm.annotation.Relationship -import org.neo4j.ogm.config.ObjectMapperFactory.objectMapper import picocli.CommandLine -@Tag("integration") -class ApplicationTest { - private fun createTranslationResult(): Pair { - val topLevel = Paths.get("src").resolve("test").resolve("resources").toAbsolutePath() - val path = topLevel.resolve("client.cpp").toAbsolutePath() - - val cmd = CommandLine(Application::class.java) - cmd.parseArgs(path.toString()) - val application = cmd.getCommand() - - val translationConfiguration = application.setupTranslationConfiguration() - val translationResult = - TranslationManager.builder().config(translationConfiguration).build().analyze().get() - return application to translationResult - } - - @Test - @Throws(InterruptedException::class) - fun testPush() { - val (application, translationResult) = createTranslationResult() - - assertEquals(32, translationResult.functions.size) - - application.pushToNeo4j(translationResult) - - val sessionAndSessionFactoryPair = application.connect() - - val session = sessionAndSessionFactoryPair.first - session.beginTransaction().use { transaction -> - val functions = session.loadAll(FunctionDeclaration::class.java) - assertNotNull(functions) +fun createTranslationResult(): Pair { + val topLevel = Paths.get("src").resolve("test").resolve("resources").toAbsolutePath() + val path = topLevel.resolve("client.cpp").toAbsolutePath() - assertEquals(32, functions.size) + val cmd = CommandLine(Application::class.java) + cmd.parseArgs(path.toString()) + val application = cmd.getCommand() - transaction.commit() - } + val translationConfiguration = application.setupTranslationConfiguration() + val translationResult = + TranslationManager.builder().config(translationConfiguration).build().analyze().get() + return application to translationResult +} - session.clear() - sessionAndSessionFactoryPair.second.close() - } +class ApplicationTest { @Test fun testSerializeCpgViaOGM() { val (application, translationResult) = createTranslationResult() - assertEquals(32, translationResult.functions.size) val (nodes, edges) = application.translateCPGToOGMBuilders(translationResult) diff --git a/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Neo4JTest.kt b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Neo4JTest.kt new file mode 100644 index 0000000000..094e18cf06 --- /dev/null +++ b/cpg-neo4j/src/test/kotlin/de/fraunhofer/aisec/cpg_vis_neo4j/Neo4JTest.kt @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2024, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg_vis_neo4j + +import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration +import de.fraunhofer.aisec.cpg.graph.functions +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNotNull + +class Neo4JTest { + @Test + @Throws(InterruptedException::class) + fun testPush() { + val (application, translationResult) = createTranslationResult() + + assertEquals(32, translationResult.functions.size) + + application.pushToNeo4j(translationResult) + + val sessionAndSessionFactoryPair = application.connect() + + val session = sessionAndSessionFactoryPair.first + session.beginTransaction().use { transaction -> + val functions = session.loadAll(FunctionDeclaration::class.java) + assertNotNull(functions) + + assertEquals(32, functions.size) + + transaction.commit() + } + + session.clear() + sessionAndSessionFactoryPair.second.close() + } +} diff --git a/cpg-neo4j/src/test/resources/log4j2.xml b/cpg-neo4j/src/test/resources/log4j2.xml new file mode 100644 index 0000000000..7f97d16dbd --- /dev/null +++ b/cpg-neo4j/src/test/resources/log4j2.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + \ No newline at end of file