From d9efef97135894018912fe69d376f8584dea6952 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20Leuth=C3=A4user?= <1417198+max-leuthaeuser@users.noreply.github.com> Date: Tue, 20 Feb 2024 15:47:41 +0100 Subject: [PATCH] [jssrc2cpg][swiftsrc2cpg] Safer offset calculation (#4208) --- .../c2cpg/io/CodeDumperFromContentTest.scala | 2 +- .../jssrc2cpg/astcreation/AstCreator.scala | 2 +- .../astcreation/AstCreatorHelper.scala | 6 +-- .../io/CodeDumperFromContentTest.scala | 20 +++++++- .../swiftsrc2cpg/astcreation/AstCreator.scala | 8 ++-- .../AstForPatternSyntaxCreator.scala | 34 ++++++++----- .../astcreation/AstForSyntaxCreator.scala | 18 +++++-- .../io/CodeDumperFromContentTest.scala | 48 +++++++++++++++---- 8 files changed, 105 insertions(+), 33 deletions(-) diff --git a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/io/CodeDumperFromContentTest.scala b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/io/CodeDumperFromContentTest.scala index 120b8d204e13..6b856dfe3803 100644 --- a/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/io/CodeDumperFromContentTest.scala +++ b/joern-cli/frontends/c2cpg/src/test/scala/io/joern/c2cpg/io/CodeDumperFromContentTest.scala @@ -65,7 +65,7 @@ class CodeDumperFromContentTest extends CCodeToCpgSuite { "Foo.cpp" ).withConfig(Config().withDisableFileContent(false)) - "allow one to dump a method node's source code from `TypeDecl.content`" in { + "allow one to dump a typedecl node's source code from `TypeDecl.content`" in { val List(content) = cpg.typeDecl.nameExact("Foo").content.l content shouldBe myClassContent } diff --git a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreator.scala index 82b04979505d..8b62de71bf60 100644 --- a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreator.scala +++ b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreator.scala @@ -264,7 +264,7 @@ class AstCreator(val config: Config, val global: Global, val parserResult: Parse for { startOffset <- start(node) endOffset <- end(node) - } yield (startOffset, endOffset) + } yield (math.max(startOffset, 0), math.min(endOffset, parserResult.fileContent.length)) } override protected def offset(node: BabelNodeInfo): Option[(Int, Int)] = { diff --git a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreatorHelper.scala b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreatorHelper.scala index aa33ef29256f..80a05bc72125 100644 --- a/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreatorHelper.scala +++ b/joern-cli/frontends/jssrc2cpg/src/main/scala/io/joern/jssrc2cpg/astcreation/AstCreatorHelper.scala @@ -72,10 +72,10 @@ trait AstCreatorHelper(implicit withSchemaValidation: ValidationMode) { this: As protected def code(node: Value): String = { nodeOffsets(node) match { - case Some((startOffset, endOffset)) - if startOffset < endOffset && startOffset >= 0 && endOffset <= parserResult.fileContent.length => + case Some((startOffset, endOffset)) => shortenCode(parserResult.fileContent.substring(startOffset, endOffset).trim) - case _ => PropertyDefaults.Code + case _ => + PropertyDefaults.Code } } diff --git a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/io/CodeDumperFromContentTest.scala b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/io/CodeDumperFromContentTest.scala index dd9f484474ac..378a362a5027 100644 --- a/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/io/CodeDumperFromContentTest.scala +++ b/joern-cli/frontends/jssrc2cpg/src/test/scala/io/joern/jssrc2cpg/io/CodeDumperFromContentTest.scala @@ -69,7 +69,25 @@ class CodeDumperFromContentTest extends JsSrc2CpgSuite { "index.js" ).withConfig(Config().withDisableFileContent(false)) - "allow one to dump a method node's source code from `TypeDecl.content`" in { + "allow one to dump a typedecl node's source code from `TypeDecl.content`" in { + val List(content) = cpg.typeDecl.nameExact("Foo").content.l + content shouldBe myClassContent + } + } + + "content with UTF8 characters" should { + val myClassContent = + """class Foo { + | // ✅ This is a comment with UTF8. + | x = 'foo'; + |}""".stripMargin + + val cpg = code(s""" + |// A comment + |$myClassContent + |""".stripMargin).withConfig(Config().withDisableFileContent(false)) + + "allow one to dump source code" in { val List(content) = cpg.typeDecl.nameExact("Foo").content.l content shouldBe myClassContent } diff --git a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstCreator.scala b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstCreator.scala index 2fdd8e2bdce3..f9f71485ae75 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstCreator.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstCreator.scala @@ -139,7 +139,7 @@ class AstCreator(val config: Config, val global: SwiftGlobal, val parserResult: for { startOffset <- node.startOffset endOffset <- node.endOffset - } yield (startOffset, endOffset) + } yield (math.max(startOffset, 0), math.min(endOffset, parserResult.fileContent.length)) } override protected def offset(node: SwiftNode): Option[(Int, Int)] = { @@ -148,10 +148,10 @@ class AstCreator(val config: Config, val global: SwiftGlobal, val parserResult: override protected def code(node: SwiftNode): String = { nodeOffsets(node) match { - case Some((startOffset, endOffset)) - if startOffset < endOffset && startOffset >= 0 && endOffset <= parserResult.fileContent.length => + case Some((startOffset, endOffset)) => shortenCode(parserResult.fileContent.substring(startOffset, endOffset).trim) - case _ => PropertyDefaults.Code + case _ => + PropertyDefaults.Code } } } diff --git a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstForPatternSyntaxCreator.scala b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstForPatternSyntaxCreator.scala index c9001e2d5112..e53ff894dd12 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstForPatternSyntaxCreator.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstForPatternSyntaxCreator.scala @@ -6,7 +6,9 @@ import io.joern.swiftsrc2cpg.parser.SwiftNodeSyntax.* import io.joern.swiftsrc2cpg.passes.Defines import io.joern.x2cpg.Ast import io.joern.x2cpg.ValidationMode +import io.shiftleft.codepropertygraph.generated.DispatchTypes import io.shiftleft.codepropertygraph.generated.EdgeTypes +import io.shiftleft.codepropertygraph.generated.Operators trait AstForPatternSyntaxCreator(implicit withSchemaValidation: ValidationMode) { this: AstCreator => @@ -22,7 +24,15 @@ trait AstForPatternSyntaxCreator(implicit withSchemaValidation: ValidationMode) Ast(identNode) } - private def astForIsTypePatternSyntax(node: IsTypePatternSyntax): Ast = notHandledYet(node) + private def astForIsTypePatternSyntax(node: IsTypePatternSyntax): Ast = { + val op = Operators.instanceOf + val typ = code(node.`type`) + val lhsNode = node.`type` + val lhsAst = Ast(literalNode(lhsNode, code(lhsNode), None).dynamicTypeHintFullName(Seq(typ))) + val callNode_ = callNode(node, code(node), op, DispatchTypes.STATIC_DISPATCH).dynamicTypeHintFullName(Seq(typ)) + callAst(callNode_, Seq(lhsAst)) + } + private def astForMissingPatternSyntax(node: MissingPatternSyntax): Ast = notHandledYet(node) private def astForTuplePatternSyntax(node: TuplePatternSyntax): Ast = notHandledYet(node) @@ -34,29 +44,31 @@ trait AstForPatternSyntaxCreator(implicit withSchemaValidation: ValidationMode) MethodScope } - val name = node.pattern match { + val (name, typeFullName) = node.pattern match { + case expr: ExpressionPatternSyntax if expr.expression.isInstanceOf[AsExprSyntax] => + val asExpr = expr.expression.asInstanceOf[AsExprSyntax] + (code(asExpr.expression), code(asExpr.`type`)) case expr: ExpressionPatternSyntax => notHandledYet(expr) - code(expr) + (code(expr), Defines.Any) case ident: IdentifierPatternSyntax => - code(ident.identifier) + (code(ident.identifier), Defines.Any) case isType: IsTypePatternSyntax => notHandledYet(isType) - code(isType) + (code(isType), Defines.Any) case missing: MissingPatternSyntax => - code(missing.placeholder) + (code(missing.placeholder), Defines.Any) case tuple: TuplePatternSyntax => notHandledYet(tuple) - code(tuple) + (code(tuple), Defines.Any) case valueBinding: ValueBindingPatternSyntax => notHandledYet(valueBinding) - code(valueBinding) + (code(valueBinding), Defines.Any) case wildcard: WildcardPatternSyntax => notHandledYet(wildcard) - generateUnusedVariableName(usedVariableNames, "wildcard") + (generateUnusedVariableName(usedVariableNames, "wildcard"), Defines.Any) } - val typeFullName = Defines.Any - val nLocalNode = localNode(node, name, name, typeFullName).order(0) + val nLocalNode = localNode(node, name, name, typeFullName).order(0) scope.addVariable(name, nLocalNode, scopeType) diffGraph.addEdge(localAstParentStack.head, nLocalNode, EdgeTypes.AST) Ast() diff --git a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstForSyntaxCreator.scala b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstForSyntaxCreator.scala index 670bf56b0d1f..fc02313f1bde 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstForSyntaxCreator.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/main/scala/io/joern/swiftsrc2cpg/astcreation/AstForSyntaxCreator.scala @@ -13,6 +13,8 @@ import io.shiftleft.codepropertygraph.generated.nodes.NewAnnotationParameter import io.shiftleft.codepropertygraph.generated.nodes.NewAnnotationParameterAssign import io.shiftleft.codepropertygraph.generated.EdgeTypes import io.shiftleft.codepropertygraph.generated.nodes.NewIdentifier +import io.shiftleft.codepropertygraph.generated.DispatchTypes +import io.shiftleft.codepropertygraph.generated.Operators trait AstForSyntaxCreator(implicit withSchemaValidation: ValidationMode) { this: AstCreator => @@ -258,10 +260,18 @@ trait AstForSyntaxCreator(implicit withSchemaValidation: ValidationMode) { this: private def astForLabeledSpecializeArgumentSyntax(node: LabeledSpecializeArgumentSyntax): Ast = notHandledYet(node) private def astForLayoutRequirementSyntax(node: LayoutRequirementSyntax): Ast = notHandledYet(node) - private def astForMatchingPatternConditionSyntax(node: MatchingPatternConditionSyntax): Ast = notHandledYet(node) - private def astForMemberBlockItemSyntax(node: MemberBlockItemSyntax): Ast = notHandledYet(node) - private def astForMemberBlockSyntax(node: MemberBlockSyntax): Ast = notHandledYet(node) - private def astForMissingSyntax(node: MissingSyntax): Ast = notHandledYet(node) + + private def astForMatchingPatternConditionSyntax(node: MatchingPatternConditionSyntax): Ast = { + val lhsAst = astForNode(node.pattern) + val rhsAst = astForNode(node.initializer.value) + val callNode_ = callNode(node, code(node), Operators.assignment, DispatchTypes.STATIC_DISPATCH) + val argAsts = List(lhsAst, rhsAst) + callAst(callNode_, argAsts) + } + + private def astForMemberBlockItemSyntax(node: MemberBlockItemSyntax): Ast = notHandledYet(node) + private def astForMemberBlockSyntax(node: MemberBlockSyntax): Ast = notHandledYet(node) + private def astForMissingSyntax(node: MissingSyntax): Ast = notHandledYet(node) private def astForMultipleTrailingClosureElementSyntax(node: MultipleTrailingClosureElementSyntax): Ast = notHandledYet(node) private def astForObjCSelectorPieceSyntax(node: ObjCSelectorPieceSyntax): Ast = notHandledYet(node) diff --git a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/io/CodeDumperFromContentTest.scala b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/io/CodeDumperFromContentTest.scala index 665b1eb46b42..f7e9863aa71c 100644 --- a/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/io/CodeDumperFromContentTest.scala +++ b/joern-cli/frontends/swiftsrc2cpg/src/test/scala/io/joern/swiftsrc2cpg/io/CodeDumperFromContentTest.scala @@ -6,22 +6,20 @@ import io.shiftleft.semanticcpg.language.* class CodeDumperFromContentTest extends SwiftSrc2CpgSuite { - private val codeString = """ - |// A comment - |func my_func(param1: Int) -> Int { - | let x: Int = foo(p: param1) - |}""".stripMargin - "dumping code from content" should { implicit val finder: NodeExtensionFinder = DefaultNodeExtensionFinder - val cpg = code(codeString, "test.swift").withConfig(Config().withDisableFileContent(false)) + val cpg = code(""" + |// A comment + |func my_func(param1: Int) -> Int { + | let x: Int = foo(p: param1) + |}""".stripMargin).withConfig(Config().withDisableFileContent(false)) "allow one to dump a method node's source code from `File.contents`" in { inside(cpg.method.nameExact("my_func").dumpRaw.l) { case content :: Nil => content.linesIterator.map(_.strip).l shouldBe List( - "func my_func(param1: Int) -> Int { /* <=== test.swift::my_func */", + "func my_func(param1: Int) -> Int { /* <=== Test0.swift::my_func */", "let x: Int = foo(p: param1)", "}" ) @@ -30,4 +28,38 @@ class CodeDumperFromContentTest extends SwiftSrc2CpgSuite { } } + "code from method content" should { + val myFuncContent = + """func my_func(param1: Int) -> Int { + | let x: Int = foo(p: param1) + |}""".stripMargin + + val cpg = code(s""" + |// A comment + |$myFuncContent + |""".stripMargin).withConfig(Config().withDisableFileContent(false)) + + "allow one to dump a method node's source code from `Method.content`" in { + val List(content) = cpg.method.nameExact("my_func").content.l + content shouldBe myFuncContent + } + } + + "code from typedecl content" should { + val myClassContent = + """class Foo { + | var x = 'foo'; + |}""".stripMargin + + val cpg = code(s""" + |// A comment + |$myClassContent + |""".stripMargin).withConfig(Config().withDisableFileContent(false)) + + "allow one to dump a typedecl node's source code from `TypeDecl.content`" in { + val List(content) = cpg.typeDecl.nameExact("Foo").content.l + content shouldBe myClassContent + } + } + }